diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td index 99cc6da0ec304..4711867e9ccac 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td @@ -328,28 +328,28 @@ def ROCDL_BarrierOp : ROCDL_Op<"barrier"> { def ROCDLGlobalBuffer : LLVM_PointerInAddressSpace<1>; def ROCDLBufferLDS : LLVM_PointerInAddressSpace<3>; -def ROCDL_BarrierInitOp : ROCDL_IntrOp<"s.barrier.init", [], [], [], 0, 0, 0, 0, [1], ["id"]>, - Arguments<(ins Arg:$ptr, I32Attr:$id)> { +def ROCDL_BarrierInitOp : ROCDL_IntrOp<"s.barrier.init", [], [], [], 0, 0, 0, 0, [1], ["memberCnt"]>, + Arguments<(ins Arg:$ptr, I32Attr:$memberCnt)> { let description = [{ Available on gfx1250+. }]; let results = (outs); - let assemblyFormat = "$ptr `,` $id attr-dict"; + let assemblyFormat = "$ptr `member_cnt` `=` $memberCnt attr-dict `:` qualified(type($ptr))"; } def ROCDL_BarrierSignalOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.signal", [], 0, [0], ["id"]>, Arguments<(ins I32Attr:$id)> { let results = (outs); - let assemblyFormat = "$id attr-dict"; + let assemblyFormat = "`id` `=` $id attr-dict"; } -def ROCDL_BarrierSignalVarOp : ROCDL_IntrOp<"s.barrier.signal.var", [], [], [], 0, 0, 0, 0, [1], ["id"]>, - Arguments<(ins Arg:$ptr, I32Attr:$id)> { +def ROCDL_BarrierSignalVarOp : ROCDL_IntrOp<"s.barrier.signal.var", [], [], [], 0, 0, 0, 0, [1], ["memberCnt"]>, + Arguments<(ins Arg:$ptr, I32Attr:$memberCnt)> { let description = [{ Available on gfx1250+. }]; let results = (outs); - let assemblyFormat = "$ptr `,` $id attr-dict"; + let assemblyFormat = "$ptr `member_cnt` `=` $memberCnt attr-dict `:` qualified(type($ptr))"; } def ROCDL_BarrierJoinOp : ROCDL_IntrOp<"s.barrier.join", [], [], [], 0>, @@ -358,7 +358,7 @@ def ROCDL_BarrierJoinOp : ROCDL_IntrOp<"s.barrier.join", [], [], [], 0>, Available on gfx1250+. }]; let results = (outs); - let assemblyFormat = "$ptr attr-dict"; + let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))"; } def ROCDL_BarrierLeaveOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.leave", [], 0, [0], ["id"]>, @@ -367,13 +367,13 @@ def ROCDL_BarrierLeaveOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.leave", [], 0, Available on gfx1250+. }]; let results = (outs); - let assemblyFormat = "$id attr-dict"; + let assemblyFormat = "`id` `=` $id attr-dict"; } def ROCDL_BarrierWaitOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.wait", [], 0, [0], ["id"]>, Arguments<(ins I16Attr:$id)> { let results = (outs); - let assemblyFormat = "$id attr-dict"; + let assemblyFormat = "`id` `=` $id attr-dict"; } def ROCDL_BarrierSignalIsfirstOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.signal.isfirst", [], 1, [0], ["id"]>, @@ -382,7 +382,7 @@ def ROCDL_BarrierSignalIsfirstOp : ROCDL_ConcreteNonMemIntrOp<"s.barrier.signal. Available on gfx1250+. }]; let results = (outs I1:$res); - let assemblyFormat = "$id attr-dict `:` type($res)"; + let assemblyFormat = "`id` `=` $id attr-dict `->` type($res)"; } def ROCDL_GetBarrierStateOp : ROCDL_ConcreteNonMemIntrOp<"s.get.barrier.state", [], 1, [0], ["id"]>, @@ -391,7 +391,7 @@ def ROCDL_GetBarrierStateOp : ROCDL_ConcreteNonMemIntrOp<"s.get.barrier.state", Available on gfx1250+. }]; let results = (outs I32:$res); - let assemblyFormat = "$id attr-dict `:` type($res)"; + let assemblyFormat = "`id` `=` $id attr-dict `->` type($res)"; } def ROCDL_GetNamedBarrierStateOp : ROCDL_ConcreteNonMemIntrOp<"s.get.named.barrier.state", [], 1, [], []>, @@ -400,7 +400,18 @@ def ROCDL_GetNamedBarrierStateOp : ROCDL_ConcreteNonMemIntrOp<"s.get.named.barri Available on gfx1250+. }]; let results = (outs I32:$res); - let assemblyFormat = "$ptr attr-dict `:` type($res)"; + let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr)) `->` type($res)"; +} + +def ROCDL_WakeupBarrierOp : ROCDL_ConcreteNonMemIntrOp<"s.wakeup.barrier", [], 0, [], []>, + Arguments<(ins Arg:$ptr)> { + let description = [{ + Wakes up waves associated with a given named barrier. Note, This op does not release waves waiting + at the barrier. It just signal other waves in the same work-group waiting on the indicated named barrier + to wake up. + Available on gfx1250+. + }]; + let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))"; } def ROCDL_WaitDscntOp: ROCDL_ConcreteNonMemIntrOp<"s.wait.dscnt", [], 0, [0], ["count"]>, diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir index 432b8876696a9..50323dc726640 100644 --- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir +++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir @@ -419,8 +419,8 @@ func.func @lds_barrier() { // GFX942-NEXT: rocdl.s.barrier // GFX10-NEXT: rocdl.s.barrier // GFX11-NEXT: rocdl.s.barrier - // GFX12-NEXT: rocdl.s.barrier.signal -1 - // GFX12-NEXT: rocdl.s.barrier.wait -1 + // GFX12-NEXT: rocdl.s.barrier.signal id = -1 + // GFX12-NEXT: rocdl.s.barrier.wait id = -1 // CHECK-NEXT: llvm.fence syncscope("workgroup") acquire {llvm.mmra = #[[$MMRA_TAG]]} amdgpu.lds_barrier func.return diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir index 3ddadde944965..750ba1c2a5eea 100644 --- a/mlir/test/Dialect/LLVMIR/rocdl.mlir +++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir @@ -1199,64 +1199,71 @@ llvm.func @rocdl.s.barrier() { llvm.func @rocdl.s.barrier.init(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.init - // CHECK: rocdl.s.barrier.init %[[PTR:.+]], 1 - rocdl.s.barrier.init %ptr, 1 + // CHECK: rocdl.s.barrier.init %{{.*}} member_cnt = 1 : !llvm.ptr<3> + rocdl.s.barrier.init %ptr member_cnt = 1 : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.signal() { // CHECK-LABEL: rocdl.s.barrier.signal - // CHECK: rocdl.s.barrier.signal -1 - rocdl.s.barrier.signal -1 + // CHECK: rocdl.s.barrier.signal id = -1 + rocdl.s.barrier.signal id = -1 llvm.return } llvm.func @rocdl.s.barrier.signal.var(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.signal.var - // CHECK: rocdl.s.barrier.signal.var %[[PTR:.+]], 1 - rocdl.s.barrier.signal.var %ptr, 1 + // CHECK: rocdl.s.barrier.signal.var %{{.*}} member_cnt = 1 : !llvm.ptr<3> + rocdl.s.barrier.signal.var %ptr member_cnt = 1 : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.join(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.join - // CHECK: rocdl.s.barrier.join %[[PTR:.+]] - rocdl.s.barrier.join %ptr + // CHECK: rocdl.s.barrier.join %{{.*}} : !llvm.ptr<3> + rocdl.s.barrier.join %ptr : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.leave() { // CHECK-LABEL: rocdl.s.barrier.leave - // CHECK: rocdl.s.barrier.leave 1 - rocdl.s.barrier.leave 1 + // CHECK: rocdl.s.barrier.leave id = 1 + rocdl.s.barrier.leave id = 1 llvm.return } llvm.func @rocdl.s.barrier.wait() { // CHECK-LABEL: rocdl.s.barrier.wait - // CHECK: rocdl.s.barrier.wait -1 - rocdl.s.barrier.wait -1 + // CHECK: rocdl.s.barrier.wait id = -1 + rocdl.s.barrier.wait id = -1 llvm.return } llvm.func @rocdl.s.barrier.signal.isfirst() { // CHECK-LABEL: rocdl.s.barrier.signal.isfirst - // CHECK: rocdl.s.barrier.signal.isfirst 1 - %0 = rocdl.s.barrier.signal.isfirst 1 : i1 + // CHECK: rocdl.s.barrier.signal.isfirst id = 1 -> i1 + %0 = rocdl.s.barrier.signal.isfirst id = 1 -> i1 llvm.return } llvm.func @rocdl.s.get.barrier.state() { // CHECK-LABEL: rocdl.s.get.barrier.state - // CHECK: rocdl.s.get.barrier.state 1 - %0 = rocdl.s.get.barrier.state 1 : i32 + // CHECK: rocdl.s.get.barrier.state id = 1 -> i32 + %0 = rocdl.s.get.barrier.state id = 1 -> i32 llvm.return } llvm.func @rocdl.s.get.named.barrier.state(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.get.named.barrier.state - // CHECK: rocdl.s.get.named.barrier.state %[[PTR:.+]] - %0 = rocdl.s.get.named.barrier.state %ptr : i32 + // CHECK: rocdl.s.get.named.barrier.state %{{.*}} : !llvm.ptr<3> -> i32 + %0 = rocdl.s.get.named.barrier.state %ptr : !llvm.ptr<3> -> i32 + llvm.return +} + +llvm.func @rocdl.s.wakeup.barrier(%ptr : !llvm.ptr<3>) { + // CHECK-LABEL: rocdl.s.wakeup.barrier + // CHECK: rocdl.s.wakeup.barrier %{{.*}} : !llvm.ptr<3> + rocdl.s.wakeup.barrier %ptr : !llvm.ptr<3> llvm.return } diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir index 5e1d857bd2a5d..1aba52f6cdaa8 100644 --- a/mlir/test/Target/LLVMIR/rocdl.mlir +++ b/mlir/test/Target/LLVMIR/rocdl.mlir @@ -253,64 +253,71 @@ llvm.func @rocdl.barrier() { llvm.func @rocdl.s.barrier.init(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.init - // CHECK: call void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) %[[PTR:.+]], i32 1) - rocdl.s.barrier.init %ptr, 1 + // CHECK: call void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) %{{.*}}, i32 1) + rocdl.s.barrier.init %ptr member_cnt = 1 : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.signal() { // CHECK-LABEL: rocdl.s.barrier.signal // CHECK-NEXT: call void @llvm.amdgcn.s.barrier.signal(i32 -1) - rocdl.s.barrier.signal -1 + rocdl.s.barrier.signal id = -1 llvm.return } llvm.func @rocdl.s.barrier.signal.var(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.signal.var - // CHECK: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) %[[PTR:.+]], i32 1) - rocdl.s.barrier.signal.var %ptr, 1 + // CHECK: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) %{{.*}}, i32 1) + rocdl.s.barrier.signal.var %ptr member_cnt = 1 : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.join(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.barrier.join - // CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) %[[PTR:.+]]) - rocdl.s.barrier.join %ptr + // CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) %{{.*}}) + rocdl.s.barrier.join %ptr : !llvm.ptr<3> llvm.return } llvm.func @rocdl.s.barrier.leave() { // CHECK-LABEL: rocdl.s.barrier.leave // CHECK: call void @llvm.amdgcn.s.barrier.leave(i16 1) - rocdl.s.barrier.leave 1 + rocdl.s.barrier.leave id = 1 llvm.return } llvm.func @rocdl.s.barrier.wait() { // CHECK-LABEL: rocdl.s.barrier.wait // CHECK-NEXT: call void @llvm.amdgcn.s.barrier.wait(i16 -1) - rocdl.s.barrier.wait -1 + rocdl.s.barrier.wait id = -1 llvm.return } llvm.func @rocdl.s.barrier.signal.isfirst() { // CHECK-LABEL: rocdl.s.barrier.signal.isfirst - // CHECK: %[[OUT:.+]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 1) - %0 = rocdl.s.barrier.signal.isfirst 1 : i1 + // CHECK: %{{.*}} = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 1) + %0 = rocdl.s.barrier.signal.isfirst id = 1 -> i1 llvm.return } llvm.func @rocdl.s.get.barrier.state() { // CHECK-LABEL: rocdl.s.get.barrier.state - // CHECK: %[[STATE:.+]] = call i32 @llvm.amdgcn.s.get.barrier.state(i32 1) - %0 = rocdl.s.get.barrier.state 1 : i32 + // CHECK: %{{.*}} = call i32 @llvm.amdgcn.s.get.barrier.state(i32 1) + %0 = rocdl.s.get.barrier.state id = 1 -> i32 llvm.return } llvm.func @rocdl.s.get.named.barrier.state(%ptr : !llvm.ptr<3>) { // CHECK-LABEL: rocdl.s.get.named.barrier.state - // CHECK: %[[STATE:.+]] = call i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3) %[[PTR:.+]]) - %0 = rocdl.s.get.named.barrier.state %ptr : i32 + // CHECK: %{{.*}} = call i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3) %{{.*}}) + %0 = rocdl.s.get.named.barrier.state %ptr : !llvm.ptr<3> -> i32 + llvm.return +} + +llvm.func @rocdl.s.wakeup.barrier(%ptr : !llvm.ptr<3>) { + // CHECK-LABEL: rocdl.s.wakeup.barrier + // CHECK: call void @llvm.amdgcn.s.wakeup.barrier(ptr addrspace(3) %{{.*}}) + rocdl.s.wakeup.barrier %ptr : !llvm.ptr<3> llvm.return }