diff --git a/configs/example/gpufs/Disjoint_VIPER.py b/configs/example/gpufs/Disjoint_VIPER.py index 8be3ff6c6b1..1529dc11c62 100644 --- a/configs/example/gpufs/Disjoint_VIPER.py +++ b/configs/example/gpufs/Disjoint_VIPER.py @@ -50,6 +50,7 @@ def __init__(self): super().__init__() def create(self, options, system, piobus, dma_devices): + self.clk_domain = system.fabric_clk # Disjoint network topology if "garnet" in options.network: self.network_cpu = DisjointGarnet(self) diff --git a/configs/example/gpufs/mi300.py b/configs/example/gpufs/mi300.py index 02db435cdd6..832ca3cbcb8 100644 --- a/configs/example/gpufs/mi300.py +++ b/configs/example/gpufs/mi300.py @@ -71,17 +71,18 @@ sh /home/gem5/load_amdgpu.sh elif [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5." - /sbin/m5 exit + # m5 exit else # Backward compatibility with old disk images (ROCm 6.1) modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2 fi +modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2 echo "Running {} {}" echo "{}" | base64 -d > myapp chmod +x myapp ./myapp {} -/sbin/m5 exit +m5 exit """ demo_runscript_with_checkpoint = """\ diff --git a/configs/example/gpufs/system/amdgpu.py b/configs/example/gpufs/system/amdgpu.py index e9a7429516a..37582d41780 100644 --- a/configs/example/gpufs/system/amdgpu.py +++ b/configs/example/gpufs/system/amdgpu.py @@ -36,7 +36,10 @@ def createGPU(system, args): n_wf=args.wfs_per_simd, cu_per_sqc=args.cu_per_sqc, timing=True, - clk_domain=system.clk_domain, + clk_domain=SrcClockDomain( + clock=args.gpu_clock, + voltage_domain=VoltageDomain(voltage=args.gpu_voltage), + ), progress_interval=args.gpu_progress_interval, ) diff --git a/configs/example/gpufs/system/system.py b/configs/example/gpufs/system/system.py index 312457a6126..31e1e5eb914 100644 --- a/configs/example/gpufs/system/system.py +++ b/configs/example/gpufs/system/system.py @@ -54,6 +54,8 @@ def makeGpuFSSystem(args): "drm_kms_helper.fbdev_emulation=0", "modprobe.blacklist=amdgpu", "modprobe.blacklist=psmouse", + # Tell linux to use MP table for PCI IRQs and not ACPI. + "pci=noacpi", ] cmdline = " ".join(boot_options) @@ -73,6 +75,13 @@ def makeGpuFSSystem(args): ) system.workload.object_file = binary(args.kernel) + # FADT pointing at a minimal DSDT. This prevents Linux from disabling + # ACPI which is needed by the WMI module which is a dependency for the + # amdgpu module. + fadt = X86ACPIFADT(dsdt=X86ACPIDSDT(), oem_id="gem5") + system.workload.acpi_description_table_pointer.rsdt.entries.append(fadt) + system.workload.acpi_description_table_pointer.xsdt.entries.append(fadt) + # Set the cache line size for the entire system. system.cache_line_size = args.cacheline_size @@ -302,6 +311,13 @@ def makeGpuFSSystem(args): pm4_proc.pio = system.iobus.mem_side_ports system_hub.pio = system.iobus.mem_side_ports + system.fabric_clk = SrcClockDomain( + clock=args.fabric_clock, voltage_domain=system.voltage_domain + ) + system.memory_clk = SrcClockDomain( + clock=args.memory_clock, voltage_domain=system.voltage_domain + ) + # Full system needs special TLBs for SQC, Scalar, and vector data ports args.full_system = True GPUTLBConfig.config_tlb_hierarchy( @@ -313,9 +329,10 @@ def makeGpuFSSystem(args): system.ruby.create(args, system, system.iobus, system._dma_ports) # Create a seperate clock domain for Ruby - system.ruby.clk_domain = SrcClockDomain( - clock=args.ruby_clock, voltage_domain=system.voltage_domain - ) + # system.ruby.clk_domain = SrcClockDomain( + # clock=args.ruby_clock, voltage_domain=system.voltage_domain + # ) + system.ruby.clk_domain = system.fabric_clk # If we are using KVM cpu, enable AVX. AVX is used in some ROCm libraries # such as rocBLAS which is used in higher level libraries like PyTorch. diff --git a/configs/ruby/GPU_VIPER.py b/configs/ruby/GPU_VIPER.py index d25bff75e67..56d01b6d52e 100644 --- a/configs/ruby/GPU_VIPER.py +++ b/configs/ruby/GPU_VIPER.py @@ -165,7 +165,7 @@ def create(self, options, ruby_system, system): self.L1cache.dataArrayBanks = options.tcp_num_banks self.L1cache.tagArrayBanks = options.tcp_num_banks self.L1cache.create(options) - self.issue_latency = 1 + self.issue_latency = options.tcp_issue_latency # TCP_Controller inherits this from RubyController self.mandatory_queue_latency = options.mandatory_queue_latency @@ -320,9 +320,11 @@ def create(self, options, ruby_system, system): self.L2cache.resourceStalls = options.no_tcc_resource_stalls self.ruby_system = ruby_system - if hasattr(options, "gpu_clock") and hasattr(options, "gpu_voltage"): + if hasattr(options, "fabric_clock") and hasattr( + options, "gpu_voltage" + ): self.clk_domain = SrcClockDomain( - clock=options.gpu_clock, + clock=options.fabric_clock, voltage_domain=VoltageDomain(voltage=options.gpu_voltage), ) @@ -424,6 +426,7 @@ def connectWireBuffers( def define_options(parser): parser.add_argument("--num-subcaches", type=int, default=4) + parser.add_argument("--tcp-issue-latency", type=int, default=1) parser.add_argument("--l3-data-latency", type=int, default=20) parser.add_argument("--l3-tag-latency", type=int, default=15) parser.add_argument("--cpu-to-dir-latency", type=int, default=120) @@ -465,7 +468,17 @@ def define_options(parser): "--TCP_latency", type=int, default=4, - help="In combination with the number of banks for the " + help="Set tcp tag access latency. " + "In combination with the number of banks for the " + "TCP, this determines how many requests can happen " + "per cycle (i.e., the bandwidth)", + ) + parser.add_argument( + "--TCP_latency_data", + type=int, + default=4, + help="Set tcp data access latency. " + "In combination with the number of banks for the " "TCP, this determines how many requests can happen " "per cycle (i.e., the bandwidth)", ) @@ -537,6 +550,8 @@ def define_options(parser): default="8", help="Data access latency in L2 cache", ) + parser.add_argument("--fabric-clock", type=str, default="1080MHz") + parser.add_argument("--memory-clock", type=str, default="1000MHz") def construct_dirs(options, system, ruby_system, network): @@ -631,7 +646,11 @@ def construct_gpudirs(options, system, ruby_system, network): xorHighBit=xor_low_bit, ) - dir_cntrl = DirCntrl(noTCCdir=True, TCC_select_num_bits=TCC_bits) + dir_cntrl = DirCntrl( + noTCCdir=True, + TCC_select_num_bits=TCC_bits, + clk_domain=system.fabric_clk, + ) dir_cntrl.create(options, [addr_range], ruby_system, system) dir_cntrl.number_of_TBEs = options.num_tbes dir_cntrl.useL3OnWT = False @@ -702,7 +721,9 @@ def construct_gpudirs(options, system, ruby_system, network): if issubclass(mem_type, DRAMInterface): if options.hbm_ctrl: mem_ctrl = m5.objects.HBMCtrl( - dram=dram_intf, dram_2=dram_intf_2 + dram=dram_intf, + dram_2=dram_intf_2, + clk_domain=system.memory_clk, ) else: mem_ctrl = m5.objects.MemCtrl(dram=dram_intf) @@ -780,7 +801,7 @@ def construct_tcps(options, system, ruby_system, network): tcp_cntrl.WB = options.WB_L1 tcp_cntrl.disableL1 = options.noL1 tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency - tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency + tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency_data exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % i) # @@ -936,7 +957,10 @@ def construct_tccs(options, system, ruby_system, network): tcc_cntrl_nodes = [] for i in range(options.num_tccs): - tcc_cntrl = TCCCntrl(l2_response_latency=options.TCC_latency) + tcc_cntrl = TCCCntrl( + l2_response_latency=options.TCC_latency, + clk_domain=system.fabric_clk, + ) tcc_cntrl.create(options, ruby_system, system) tcc_cntrl.l2_request_latency = options.gpu_to_dir_latency tcc_cntrl.l2_response_latency = options.TCC_latency diff --git a/src/arch/amdgpu/vega/gpu_decoder.cc b/src/arch/amdgpu/vega/gpu_decoder.cc index 4574e88a98e..6f26338bb7c 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.cc +++ b/src/arch/amdgpu/vega/gpu_decoder.cc @@ -512,10 +512,10 @@ IsaDecodeMethod Decoder::tableDecodePrimary[] = { &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, diff --git a/src/arch/amdgpu/vega/gpu_decoder.hh b/src/arch/amdgpu/vega/gpu_decoder.hh index e40db172aa1..f8e4d88ab6f 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.hh +++ b/src/arch/amdgpu/vega/gpu_decoder.hh @@ -1782,7 +1782,9 @@ struct InFmt_DS unsigned int OFFSET1 : 8; unsigned int GDS : 1; unsigned int OP : 8; - unsigned int pad_25 : 1; + // CDNA ACC selects the AGPR bank for DS data registers. ADDR remains + // a VGPR. + unsigned int ACC : 1; unsigned int ENCODING : 6; }; diff --git a/src/arch/amdgpu/vega/insts/ds.cc b/src/arch/amdgpu/vega/insts/ds.cc index 698128fe5ab..f155269aaec 100644 --- a/src/arch/amdgpu/vega/insts/ds.cc +++ b/src/arch/amdgpu/vega/insts/ds.cc @@ -67,7 +67,7 @@ Inst_DS__DS_ADD_U32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); @@ -302,7 +302,7 @@ Inst_DS__DS_OR_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); @@ -404,9 +404,10 @@ Inst_DS__DS_WRITE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data(gpuDynInst, extData.DATA0); + ConstVecOperandU32 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -420,6 +421,8 @@ Inst_DS__DS_WRITE_B32::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -467,10 +470,12 @@ Inst_DS__DS_WRITE2_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -487,6 +492,8 @@ Inst_DS__DS_WRITE2_B32::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -533,10 +540,12 @@ Inst_DS__DS_WRITE2ST64_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -553,6 +562,8 @@ Inst_DS__DS_WRITE2ST64_B32::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -719,7 +730,7 @@ Inst_DS__DS_ADD_F32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandF32 data(gpuDynInst, extData.DATA0); @@ -780,7 +791,7 @@ Inst_DS__DS_WRITE_B8::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU8 data(gpuDynInst, extData.DATA0); @@ -796,6 +807,8 @@ Inst_DS__DS_WRITE_B8::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -841,7 +854,7 @@ Inst_DS__DS_WRITE_B8_D16_HI::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); @@ -857,6 +870,8 @@ Inst_DS__DS_WRITE_B8_D16_HI::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -902,7 +917,7 @@ Inst_DS__DS_WRITE_B16::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU16 data(gpuDynInst, extData.DATA0); @@ -918,6 +933,8 @@ Inst_DS__DS_WRITE_B16::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -963,7 +980,7 @@ Inst_DS__DS_WRITE_B16_D16_HI::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); @@ -979,6 +996,8 @@ Inst_DS__DS_WRITE_B16_D16_HI::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1026,7 +1045,7 @@ Inst_DS__DS_ADD_RTN_U32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); @@ -1388,7 +1407,7 @@ Inst_DS__DS_CMPST_RTN_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 src(gpuDynInst, extData.DATA1); @@ -1582,13 +1601,15 @@ Inst_DS__DS_READ_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1605,7 +1626,8 @@ Inst_DS__DS_READ_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU32 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -1646,13 +1668,15 @@ Inst_DS__DS_READ2_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1668,8 +1692,10 @@ Inst_DS__DS_READ2_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -1713,13 +1739,15 @@ Inst_DS__DS_READ2ST64_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1779,13 +1807,15 @@ Inst_DS__DS_READ_I8::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1842,13 +1872,15 @@ Inst_DS__DS_READ_U8::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -1925,13 +1957,15 @@ Inst_DS__DS_READ_U16::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute void @@ -1987,13 +2021,15 @@ Inst_DS__DS_READ_U16_D16::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute void @@ -2050,13 +2086,15 @@ Inst_DS__DS_READ_U16_D16_HI::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(1); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute void @@ -2119,7 +2157,7 @@ Inst_DS__DS_SWIZZLE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); VecOperandU32 vdst(gpuDynInst, extData.VDST); @@ -2258,7 +2296,7 @@ Inst_DS__DS_PERMUTE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); VecOperandU32 vdst(gpuDynInst, extData.VDST); @@ -2348,7 +2386,7 @@ Inst_DS__DS_BPERMUTE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU32 data(gpuDynInst, extData.DATA0); VecOperandU32 vdst(gpuDynInst, extData.VDST); @@ -2437,7 +2475,7 @@ Inst_DS__DS_ADD_U64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); ConstVecOperandU64 data(gpuDynInst, extData.DATA0); @@ -2730,9 +2768,10 @@ Inst_DS__DS_WRITE_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data(gpuDynInst, extData.DATA0); + ConstVecOperandU64 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -2746,6 +2785,8 @@ Inst_DS__DS_WRITE_B64::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -2793,10 +2834,12 @@ Inst_DS__DS_WRITE2_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -2813,6 +2856,8 @@ Inst_DS__DS_WRITE2_B64::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -2859,10 +2904,12 @@ Inst_DS__DS_WRITE2ST64_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -2879,6 +2926,8 @@ Inst_DS__DS_WRITE2ST64_B64::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -3425,13 +3474,15 @@ Inst_DS__DS_READ_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -3448,7 +3499,8 @@ Inst_DS__DS_READ_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU64 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -3489,13 +3541,15 @@ Inst_DS__DS_READ2_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -3511,8 +3565,10 @@ Inst_DS__DS_READ2_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst0(gpuDynInst, extData.VDST); - VecOperandU64 vdst1(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU64 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU64 vdst1(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -3556,13 +3612,15 @@ Inst_DS__DS_READ2ST64_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4499,11 +4557,12 @@ Inst_DS__DS_WRITE_B96::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); addr.read(); data0.read(); @@ -4523,6 +4582,8 @@ Inst_DS__DS_WRITE_B96::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(3); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4561,12 +4622,13 @@ Inst_DS__DS_WRITE_B128::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); - ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + 3); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); + ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + acc + 3); addr.read(); data0.read(); @@ -4589,6 +4651,8 @@ Inst_DS__DS_WRITE_B128::execute(GPUDynInstPtr gpuDynInst) } } + gpuDynInst->numDstScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4626,13 +4690,15 @@ Inst_DS__DS_READ_B96::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(3); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4649,9 +4715,11 @@ Inst_DS__DS_READ_B96::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B96::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4689,13 +4757,15 @@ Inst_DS__DS_READ_B128::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(4); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4712,10 +4782,12 @@ Inst_DS__DS_READ_B128::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B128::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); - VecOperandU32 vdst3(gpuDynInst, extData.VDST + 3); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); + VecOperandU32 vdst3(gpuDynInst, extData.VDST + acc + 3); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4759,13 +4831,15 @@ Inst_DS__DS_READ_B64_TR_B4::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -4888,13 +4962,15 @@ Inst_DS__DS_READ_B96_TR_B6::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -5036,13 +5112,15 @@ Inst_DS__DS_READ_B64_TR_B8::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute @@ -5151,13 +5229,15 @@ Inst_DS__DS_READ_B64_TR_B16::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->execUnitId = wf->execUnitId; gpuDynInst->latency.init(gpuDynInst->computeUnit()); gpuDynInst->latency.set( - gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); + gpuDynInst->computeUnit()->cyclesToTicks(Cycles(60))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); addr.read(); calcAddr(gpuDynInst, addr); + gpuDynInst->numSrcScalarDWords(2); + gpuDynInst->computeUnit()->localMemoryPipe.issueRequest(gpuDynInst); } // execute diff --git a/src/arch/amdgpu/vega/insts/op_encodings.cc b/src/arch/amdgpu/vega/insts/op_encodings.cc index 2000d7c63eb..6c04a283a03 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.cc +++ b/src/arch/amdgpu/vega/insts/op_encodings.cc @@ -1345,6 +1345,8 @@ Inst_DS::initOperandInfo() for (opIdx = 0; opIdx < numSrcRegOperands(); opIdx++) { srcOps.emplace_back(srcs[opIdx], getOperandSize(opIdx), true, false, true, false); + if (instData.ACC && opIdx > 0) + srcOps.back().setAccum(); } if (numDstRegOperands()) { @@ -1352,6 +1354,8 @@ Inst_DS::initOperandInfo() int reg = extData.VDST; dstOps.emplace_back(reg, getOperandSize(opIdx), false, false, true, false); + if (instData.ACC) + dstOps.back().setAccum(); } assert(srcOps.size() == numSrcRegOperands()); diff --git a/src/arch/amdgpu/vega/insts/op_encodings.hh b/src/arch/amdgpu/vega/insts/op_encodings.hh index 96fd9fd4a98..40c68f2b6b4 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.hh +++ b/src/arch/amdgpu/vega/insts/op_encodings.hh @@ -983,6 +983,14 @@ class Inst_DS : public VEGAGPUStaticInst void initOperandInfo() override; protected: + // CDNA ACC data registers are mapped into the AGPR window of the + // unified vector register file. ADDR remains an ordinary VGPR. + unsigned + accDataOffset(Wavefront *wf) const + { + return instData.ACC ? wf->accumOffset : 0; + } + template void initMemRead(GPUDynInstPtr gpuDynInst, Addr offset) diff --git a/src/gpu-compute/global_memory_pipeline.cc b/src/gpu-compute/global_memory_pipeline.cc index 24960bd0609..5df36a2f27d 100644 --- a/src/gpu-compute/global_memory_pipeline.cc +++ b/src/gpu-compute/global_memory_pipeline.cc @@ -250,12 +250,29 @@ GlobalMemPipeline::exec() GPUDynInstPtr GlobalMemPipeline::getNextReadyResp() { - if (!gmOrderedRespBuffer.empty()) { - auto mem_req = gmOrderedRespBuffer.begin(); + auto l_hash = [](const std::pair& p) -> std::size_t { + return (static_cast(p.first) << 8) | p.second; + }; + std::unordered_set, decltype(l_hash)> + wf_set(0, l_hash); - if (mem_req->second.second) { - return mem_req->second.first; + // Find one wavefront-level oldest completed request + if (!gmOrderedRespBuffer.empty()) { + for (auto it = gmOrderedRespBuffer.begin(); + it != gmOrderedRespBuffer.end(); ++it) { + Wavefront *wf = it->second.first->wavefront(); + std::pair wf_id(wf->simdId, wf->wfSlotId); + // If the request is done and we haven't seen this wavefront yet, + // return it + if (it->second.second) { + if (wf_set.find(wf_id) == wf_set.end()) { + return it->second.first; + } + } + else + wf_set.insert(wf_id); } + } return nullptr; @@ -274,9 +291,9 @@ GlobalMemPipeline::completeRequest(GPUDynInstPtr gpuDynInst) // we should only pop the oldest requst, and it // should be marked as done if we are here - assert(gmOrderedRespBuffer.begin()->first == gpuDynInst->seqNum()); - assert(gmOrderedRespBuffer.begin()->second.first == gpuDynInst); - assert(gmOrderedRespBuffer.begin()->second.second); + // assert(gmOrderedRespBuffer.begin()->first == gpuDynInst->seqNum()); + // assert(gmOrderedRespBuffer.begin()->second.first == gpuDynInst); + // assert(gmOrderedRespBuffer.begin()->second.second); // remove this instruction from the buffer by its // unique seq ID gmOrderedRespBuffer.erase(gpuDynInst->seqNum()); diff --git a/src/gpu-compute/gpu_dyn_inst.cc b/src/gpu-compute/gpu_dyn_inst.cc index 41894f9809f..5071f30bb87 100644 --- a/src/gpu-compute/gpu_dyn_inst.cc +++ b/src/gpu-compute/gpu_dyn_inst.cc @@ -229,6 +229,18 @@ GPUDynInst::numDstScalarDWords() return _staticInst->numDstScalarDWords(); } +void +GPUDynInst::numSrcScalarDWords(int numDWords) +{ + _staticInst->numSrcScalarDWords(numDWords); +} + +void +GPUDynInst::numDstScalarDWords(int numDWords) +{ + _staticInst->numDstScalarDWords(numDWords); +} + int GPUDynInst::maxOperandSize() { diff --git a/src/gpu-compute/gpu_dyn_inst.hh b/src/gpu-compute/gpu_dyn_inst.hh index 9cbf8b8e6ab..e62b1c4f5a1 100644 --- a/src/gpu-compute/gpu_dyn_inst.hh +++ b/src/gpu-compute/gpu_dyn_inst.hh @@ -188,6 +188,9 @@ class GPUDynInst : public GPUExecContext int numSrcScalarDWords(); int numDstScalarDWords(); + void numSrcScalarDWords(int numDWords); + void numDstScalarDWords(int numDwords); + int maxOperandSize(); int getNumOperands() const; diff --git a/src/gpu-compute/gpu_static_inst.cc b/src/gpu-compute/gpu_static_inst.cc index a48b57ff429..0b28adf7551 100644 --- a/src/gpu-compute/gpu_static_inst.cc +++ b/src/gpu-compute/gpu_static_inst.cc @@ -71,9 +71,30 @@ GPUStaticInst::generateVirtToPhysMap(Wavefront *wf, ComputeUnit *cu, int num_dwords = op.sizeInDWords(); int virt_idx = op.registerIndex(wf->reservedScalarRegs); + // CDNA ACC bit: the operand's data registers live in the accumulator + // (AGPR) window of the unified VRF, which begins at wf->accumOffset. + // KEEP IN SYNC with Inst_DS::accDataOffset() (op_encodings.hh), which + // applies the identical offset on the data path during execute(). + if (op.isAccum()) { + virt_idx += wf->accumOffset; + } + int phys_idx = -1; for (int i = 0; i < num_dwords; i++) { if (opType == OpType::SRC_VEC || opType == OpType::DST_VEC) { + const int this_virt_idx = virt_idx + i; + panic_if(this_virt_idx >= wf->reservedVectorRegs, + "%s maps %s vector %s operand out of reserved VGPR " + "range: virt_idx=%d raw_reg=%d base_reg=%d dword=%d/%d " + "reserved=%d accum_offset=%u is_accum=%d " + "reserved_sgprs=%d\n", + disassemble().c_str(), + opType == OpType::SRC_VEC ? "src" : "dst", + op.isAccum() ? "AGPR" : "VGPR", + this_virt_idx, op.rawRegisterIndex(), + op.registerIndex(wf->reservedScalarRegs), i, num_dwords, + wf->reservedVectorRegs, wf->accumOffset, op.isAccum(), + wf->reservedScalarRegs); phys_idx = cu->registerManager->mapVgpr(wf, virt_idx + i); } else { assert(opType == OpType::SRC_SCALAR || @@ -219,6 +240,18 @@ GPUStaticInst::numDstScalarDWords() return dstScalarDWords; } +void +GPUStaticInst::numSrcScalarDWords(int numDWords) +{ + srcScalarDWords = numDWords; +} + +void +GPUStaticInst::numDstScalarDWords(int numDWords) +{ + dstScalarDWords = numDWords; +} + int GPUStaticInst::maxOperandSize() { diff --git a/src/gpu-compute/gpu_static_inst.hh b/src/gpu-compute/gpu_static_inst.hh index 028752fb741..a325b034396 100644 --- a/src/gpu-compute/gpu_static_inst.hh +++ b/src/gpu-compute/gpu_static_inst.hh @@ -133,6 +133,10 @@ class GPUStaticInst : public GPUStaticInstFlags int numSrcScalarDWords(); int numDstScalarDWords(); + void numSrcScalarDWords(int numDWords); + void numDstScalarDWords(int numDwords); + + int maxOperandSize(); virtual int diff --git a/src/gpu-compute/hsa_queue_entry.hh b/src/gpu-compute/hsa_queue_entry.hh index f5e1d833b00..5385e8f664c 100644 --- a/src/gpu-compute/hsa_queue_entry.hh +++ b/src/gpu-compute/hsa_queue_entry.hh @@ -104,8 +104,9 @@ class HSAQueueEntry // LLVM docs: https://www.llvm.org/docs/AMDGPUUsage.html // #code-object-v3-kernel-descriptor // - // Currently, gem5 supported gfx version use a multiplier of 8. The - // only exception is gfx900 (Vega10). + // CDNA's VGPR field is encoded from LLVM's total unified allocation: + // alignTo(num_arch_vgprs, 4) + num_agprs. ACCUM_OFFSET only identifies + // where AGPR operands begin; it must not be added to this allocation. if (gfx_version == GfxVersion::gfx90a || gfx_version == GfxVersion::gfx942 || gfx_version == GfxVersion::gfx950) { diff --git a/src/gpu-compute/lds_state.cc b/src/gpu-compute/lds_state.cc index 9f7e417c090..5f2b15eb66b 100644 --- a/src/gpu-compute/lds_state.cc +++ b/src/gpu-compute/lds_state.cc @@ -109,8 +109,9 @@ LdsState::countBankConflicts(GPUDynInstPtr gpuDynInst, // if the wavefront size is larger than the number of LDS banks, we // need to iterate over all work items to calculate the total // number of bank conflicts - int groups = - (parent->wfSize() > numBanks) ? (parent->wfSize() / numBanks) : 1; + int groups = (parent->wfSize() > numBanks) ? + (parent->wfSize() / numBanks) : 1; + for (int i = 0; i < groups; i++) { // Address Array holding all the work item addresses of an instruction std::vector addr_array; @@ -142,7 +143,10 @@ LdsState::countBankConflicts(GPUDynInstPtr gpuDynInst, // calculate bank conflicts for (int j = 0; j < numBanks; ++j) { if (addr_array[j] != std::numeric_limits::max()) { - int bankId = addr_array[j] % banks; + // Determine the bank ID for this address + // Each bank is 4 bytes wide, so divide address by 4 + // to get the "bank address" + int bankId = (addr_array[j]/4) % banks; bank[bankId]++; max_bank = std::max(max_bank, bank[bankId]); // Count the number of LDS banks accessed. @@ -197,12 +201,17 @@ LdsState::processPacket(PacketPtr packet) : (dynInst->isStore()) ? parent->storeBusLength() : parent->loadBusLength(); // delay for accessing the LDS + busLength = (dynInst->isLoad()) ? + (busLength/4)*dynInst->numSrcScalarDWords() : + (dynInst->isStore()) ? (busLength/4)*dynInst->numDstScalarDWords() : + (busLength/4)*dynInst->numSrcScalarDWords(); Tick processingTime = parent->cyclesToTicks(Cycles(bankConflicts * bankConflictPenalty)) + parent->cyclesToTicks(Cycles(busLength)); // choose (delay + last packet in queue) or (now + delay) as the time to // return this Tick doneAt = earliestReturnTime() + processingTime; + //Tick doneAt = curTick() + processingTime; // then store it for processing return returnQueuePush(std::make_pair(doneAt, packet)); } diff --git a/src/gpu-compute/operand_info.hh b/src/gpu-compute/operand_info.hh index e28f2c2ddd7..05d13a1355f 100644 --- a/src/gpu-compute/operand_info.hh +++ b/src/gpu-compute/operand_info.hh @@ -157,6 +157,9 @@ class OperandInfo return flags.isSet(FLAT); } + void setAccum() { flags.set(ACCUM); } + bool isAccum() const { return flags.isSet(ACCUM); } + void setVirtToPhysMapping(std::vector v, std::vector p) { @@ -234,7 +237,10 @@ class OperandInfo CONSTANT = 0x00000100, // If the constant is positive or negative - POS_CONST = 0x00000200 + POS_CONST = 0x00000200, + + // Operand resides in the CDNA accumulator (AGPR) window. + ACCUM = 0x00000400 }; Flags flags;