From 227e84126dd81d438f203fb27ace508194f54be8 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 12 Jan 2023 10:16:25 -0800 Subject: [PATCH 01/10] Ensure isSimdAsHWIntrinsic is false for Vector64/128/256.WithElement --- src/coreclr/jit/hwintrinsicarm64.cpp | 2 +- src/coreclr/jit/hwintrinsicxarch.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 5a7a93b6ccf79..aca202514b3ad 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1738,7 +1738,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* vectorOp = impSIMDPopStack(getSIMDTypeForSize(simdSize)); retNode = gtNewSimdWithElementNode(retType, vectorOp, indexOp, valueOp, simdBaseJitType, simdSize, - /* isSimdAsHWIntrinsic */ true); + /* isSimdAsHWIntrinsic */ false); break; } diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index af92042ce0181..ea412169ce5b8 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -2307,7 +2307,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* vectorOp = impSIMDPopStack(getSIMDTypeForSize(simdSize)); retNode = gtNewSimdWithElementNode(retType, vectorOp, indexOp, valueOp, simdBaseJitType, simdSize, - /* isSimdAsHWIntrinsic */ true); + /* isSimdAsHWIntrinsic */ false); break; } From ce7c878c5bcf7e93bd279ea980f186a0702e0428 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 12 Jan 2023 11:09:59 -0800 Subject: [PATCH 02/10] Ensure AsVector2 and AsVector3 are intrinsic --- src/coreclr/jit/hwintrinsicarm64.cpp | 22 ++++++++++ src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 19 +++++--- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 2 + src/coreclr/jit/hwintrinsiclistarm64.h | 2 + src/coreclr/jit/hwintrinsiclistxarch.h | 4 +- src/coreclr/jit/hwintrinsicxarch.cpp | 21 +++------ src/coreclr/jit/lsraarm64.cpp | 43 ++++++++++++++----- src/coreclr/jit/lsraxarch.cpp | 2 + .../System/Runtime/Intrinsics/Vector128.cs | 2 + 9 files changed, 85 insertions(+), 32 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index aca202514b3ad..92c348a0fddc7 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -414,6 +414,28 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_AsVector2: + { + assert(sig->numArgs == 1); + assert((simdSize == 16) && (simdBaseType == TYP_FLOAT)); + assert(retType == TYP_SIMD8); + + op1 = impSIMDPopStack(TYP_SIMD16); + retNode = gtNewSimdHWIntrinsicNode(retType, retNode, NI_Vector128_GetLower, simdBaseJitType, simdSize); + break; + } + + case NI_Vector128_AsVector3: + { + assert(sig->numArgs == 1); + assert((simdSize == 16) && (simdBaseType == TYP_FLOAT)); + assert(retType == TYP_SIMD12); + + op1 = impSIMDPopStack(TYP_SIMD16); + retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseJitType, simdSize); + break; + } + case NI_Vector64_BitwiseAnd: case NI_Vector128_BitwiseAnd: case NI_Vector64_op_BitwiseAnd: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6d9667166b545..406aa2fad5d36 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -955,18 +955,27 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Vector128_AsVector3: + { + // AsVector3 can be a no-op when it's already in the right register, otherwise + // we just need to move the value over. Vector3 operations will themselves mask + // out the upper element when it's relevant, so it's not worth us spending extra + // cycles doing so here. + + GetEmitter()->emitIns_Mov(ins, emitSize, targetReg, op1Reg, /* canSkip */ true); + break; + } + case NI_Vector64_ToScalar: case NI_Vector128_ToScalar: { - const ssize_t indexValue = 0; - - // no-op if vector is float/double, targetReg == op1Reg and fetching for 0th index. - if ((varTypeIsFloating(intrin.baseType) && (targetReg == op1Reg) && (indexValue == 0))) + if ((varTypeIsFloating(intrin.baseType) && (targetReg == op1Reg))) { + // no-op if vector is float/double and targetReg == op1Reg break; } - GetEmitter()->emitIns_R_R_I(ins, emitTypeSize(intrin.baseType), targetReg, op1Reg, indexValue, + GetEmitter()->emitIns_R_R_I(ins, emitTypeSize(intrin.baseType), targetReg, op1Reg, /* imm */ 0, INS_OPTS_NONE); } break; diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index ac769aeb87c3e..cb114b2d19701 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -1075,6 +1075,8 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Vector128_AsVector2: + case NI_Vector128_AsVector3: case NI_Vector128_ToScalar: case NI_Vector256_ToScalar: { diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 123149ec38c7c..bdc7602fa2bb4 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -131,6 +131,8 @@ HARDWARE_INTRINSIC(Vector128, AsUInt16, HARDWARE_INTRINSIC(Vector128, AsUInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector128, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 013213fecbca9..80fb3d48ef824 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -42,8 +42,8 @@ HARDWARE_INTRINSIC(Vector128, AsUInt16, HARDWARE_INTRINSIC(Vector128, AsUInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsdsse2, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector128, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index ea412169ce5b8..25985e7f8a090 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -581,6 +581,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsUInt16: case NI_Vector128_AsUInt32: case NI_Vector128_AsUInt64: + case NI_Vector128_AsVector4: case NI_Vector256_As: case NI_Vector256_AsByte: case NI_Vector256_AsDouble: @@ -632,22 +633,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsVector2: case NI_Vector128_AsVector3: { - // TYP_SIMD8 and TYP_SIMD12 currently only expose "safe" versions - // which zero the upper elements and so are implemented in managed. - unreached(); - } - - case NI_Vector128_AsVector4: - { - // We fold away the cast here, as it only exists to satisfy - // the type system. It is safe to do this here since the retNode type - // and the signature return type are both the same TYP_SIMD or the - // return type is a smaller TYP_SIMD that shares the same register. - - retNode = impSIMDPopStack(retType, /* expectAddr: */ false, sig->retTypeClass); - SetOpLclRelatedToSIMDIntrinsic(retNode); - assert(retNode->gtType == getSIMDTypeForSize(getSIMDTypeSizeInBytes(sig->retTypeSigClass))); + assert(sig->numArgs == 1); + assert((simdSize == 16) && (simdBaseType == TYP_FLOAT)); + assert((retType == TYP_SIMD8) || (retType == TYP_SIMD12)); + op1 = impSIMDPopStack(TYP_SIMD16); + retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseJitType, simdSize); break; } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index bb68132c80259..f8e5e2fec3548 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -933,17 +933,40 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { bool simdRegToSimdRegMove = false; - if ((intrin.id == NI_Vector64_CreateScalarUnsafe) || (intrin.id == NI_Vector128_CreateScalarUnsafe)) + switch (intrin.id) { - simdRegToSimdRegMove = varTypeIsFloating(intrin.op1); - } - else if (intrin.id == NI_AdvSimd_Arm64_DuplicateToVector64) - { - simdRegToSimdRegMove = (intrin.op1->TypeGet() == TYP_DOUBLE); - } - else if ((intrin.id == NI_Vector64_ToScalar) || (intrin.id == NI_Vector128_ToScalar)) - { - simdRegToSimdRegMove = varTypeIsFloating(intrinsicTree); + case NI_Vector64_CreateScalarUnsafe: + case NI_Vector128_CreateScalarUnsafe: + { + simdRegToSimdRegMove = varTypeIsFloating(intrin.op1); + break; + } + + case NI_AdvSimd_Arm64_DuplicateToVector64: + { + simdRegToSimdRegMove = (intrin.op1->TypeGet() == TYP_DOUBLE); + break; + } + + case NI_Vector64_ToScalar: + case NI_Vector128_ToScalar: + { + simdRegToSimdRegMove = varTypeIsFloating(intrinsicTree); + break; + } + + case NI_Vector64_ToVector128Unsafe: + case NI_Vector128_AsVector3: + case NI_Vector128_GetLower: + { + simdRegToSimdRegMove = true; + break; + } + + default: + { + break; + } } // If we have an RMW intrinsic or an intrinsic with simple move semantic between two SIMD registers, diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index c747fddaf842e..c0fd6030c2880 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -2057,6 +2057,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou break; } + case NI_Vector128_AsVector2: + case NI_Vector128_AsVector3: case NI_Vector128_ToVector256: case NI_Vector128_ToVector256Unsafe: case NI_Vector256_GetLower: diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index 88db7bd90f960..cdbedf9cadde4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -276,6 +276,7 @@ public static Vector128 AsVector128(this Vector value) /// Reinterprets a as a new . /// The vector to reinterpret. /// reinterpreted as a new . + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector2 AsVector2(this Vector128 value) { @@ -286,6 +287,7 @@ public static Vector2 AsVector2(this Vector128 value) /// Reinterprets a as a new . /// The vector to reinterpret. /// reinterpreted as a new . + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector3 AsVector3(this Vector128 value) { From 35af3f94a1c60cb7166f0756ba14f35135b5a8f8 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 12 Jan 2023 11:41:31 -0800 Subject: [PATCH 03/10] Ensure AsVector128(Vector2) and AsVector128(Vector3) are handled as intrinsic --- src/coreclr/jit/hwintrinsicarm64.cpp | 82 ++++++++++++++++++- src/coreclr/jit/hwintrinsicxarch.cpp | 51 +++++++++++- .../System/Runtime/Intrinsics/Vector128.cs | 2 + 3 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 92c348a0fddc7..ec94d13d8c715 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -399,7 +399,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsUInt64: case NI_Vector128_AsVector: case NI_Vector128_AsVector4: - case NI_Vector128_AsVector128: { assert(!sig->hasThis()); assert(numArgs == 1); @@ -436,6 +435,87 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_AsVector128: + { + assert(!sig->hasThis()); + assert(numArgs == 1); + assert(retType == TYP_SIMD16); + + switch (getSIMDTypeForSize(simdSize)) + { + case TYP_SIMD8: + { + assert((simdSize == 8) && (simdBaseType == TYP_FLOAT)); + + op1 = impSIMDPopStack(TYP_SIMD8); + + if (op1->IsCnsVec()) + { + GenTreeVecCon* vecCon = op1->AsVecCon(); + vecCon->gtType = TYP_SIMD16; + + vecCon->gtSimd16Val.f32[2] = 0.0f; + vecCon->gtSimd16Val.f32[3] = 0.0f; + + return vecCon; + } + + GenTree* idx = gtNewIconNode(2, TYP_INT); + GenTree* zero = gtNewZeroConNode(TYP_FLOAT); + op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + + idx = gtNewIconNode(3, TYP_INT); + zero = gtNewZeroConNode(TYP_FLOAT); + retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + + break; + } + + case TYP_SIMD12: + { + assert((simdSize == 12) && (simdBaseType == TYP_FLOAT)); + + op1 = impSIMDPopStack(TYP_SIMD12); + + if (op1->IsCnsVec()) + { + GenTreeVecCon* vecCon = op1->AsVecCon(); + vecCon->gtType = TYP_SIMD16; + + vecCon->gtSimd16Val.f32[3] = 0.0f; + return vecCon; + } + + GenTree* idx = gtNewIconNode(3, TYP_INT); + GenTree* zero = gtNewZeroConNode(TYP_FLOAT); + retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + break; + } + + case TYP_SIMD16: + { + // We fold away the cast here, as it only exists to satisfy + // the type system. It is safe to do this here since the retNode type + // and the signature return type are both the same TYP_SIMD. + + retNode = impSIMDPopStack(retType, /* expectAddr: */ false, sig->retTypeClass); + SetOpLclRelatedToSIMDIntrinsic(retNode); + assert(retNode->gtType == getSIMDTypeForSize(getSIMDTypeSizeInBytes(sig->retTypeSigClass))); + break; + } + + default: + { + unreached(); + } + } + + break; + } + case NI_Vector64_BitwiseAnd: case NI_Vector128_BitwiseAnd: case NI_Vector64_op_BitwiseAnd: diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 25985e7f8a090..0299efed158c0 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -645,6 +645,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsVector128: { assert(sig->numArgs == 1); + assert(retType == TYP_SIMD16); assert(HWIntrinsicInfo::BaseTypeFromFirstArg(intrinsic)); CorInfoType op1SimdBaseJitType = @@ -655,11 +656,55 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, switch (getSIMDTypeForSize(simdSize)) { case TYP_SIMD8: + { + assert((simdSize == 8) && (simdBaseType == TYP_FLOAT)); + + op1 = impSIMDPopStack(TYP_SIMD8); + + if (op1->IsCnsVec()) + { + GenTreeVecCon* vecCon = op1->AsVecCon(); + vecCon->gtType = TYP_SIMD16; + + vecCon->gtSimd16Val.f32[2] = 0.0f; + vecCon->gtSimd16Val.f32[3] = 0.0f; + + return vecCon; + } + + GenTree* idx = gtNewIconNode(2, TYP_INT); + GenTree* zero = gtNewZeroConNode(TYP_FLOAT); + op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + + idx = gtNewIconNode(3, TYP_INT); + zero = gtNewZeroConNode(TYP_FLOAT); + retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + + break; + } + case TYP_SIMD12: { - // TYP_SIMD8 and TYP_SIMD12 currently only expose "safe" versions - // which zero the upper elements and so are implemented in managed. - unreached(); + assert((simdSize == 12) && (simdBaseType == TYP_FLOAT)); + + op1 = impSIMDPopStack(TYP_SIMD12); + + if (op1->IsCnsVec()) + { + GenTreeVecCon* vecCon = op1->AsVecCon(); + vecCon->gtType = TYP_SIMD16; + + vecCon->gtSimd16Val.f32[3] = 0.0f; + return vecCon; + } + + GenTree* idx = gtNewIconNode(3, TYP_INT); + GenTree* zero = gtNewZeroConNode(TYP_FLOAT); + retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, + /* isSimdAsHWIntrinsic */ false); + break; } case TYP_SIMD16: diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index cdbedf9cadde4..de163d2550ee6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -237,6 +237,7 @@ public static Vector128 AsUInt64(this Vector128 vector) /// Reinterprets a as a new . /// The vector to reinterpret. /// reinterpreted as a new . + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 AsVector128(this Vector2 value) => new Vector4(value, 0.0f, 0.0f).AsVector128(); @@ -244,6 +245,7 @@ public static Vector128 AsVector128(this Vector2 value) /// Reinterprets a as a new . /// The vector to reinterpret. /// reinterpreted as a new . + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 AsVector128(this Vector3 value) => new Vector4(value, 0.0f).AsVector128(); From dc793d249adfc234e82dbb9a5b3693b017c50122 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 12 Jan 2023 11:48:37 -0800 Subject: [PATCH 04/10] Ensure Vector64/128/256.AsNInt and AsNUInt are handled as intrinsic --- src/coreclr/jit/hwintrinsicarm64.cpp | 4 ++++ src/coreclr/jit/hwintrinsiclistarm64.h | 4 ++++ src/coreclr/jit/hwintrinsiclistxarch.h | 4 ++++ src/coreclr/jit/hwintrinsicxarch.cpp | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index ec94d13d8c715..8a5e80ca28bdb 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -381,6 +381,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector64_AsInt16: case NI_Vector64_AsInt32: case NI_Vector64_AsInt64: + case NI_Vector64_AsNInt: + case NI_Vector64_AsNUInt: case NI_Vector64_AsSByte: case NI_Vector64_AsSingle: case NI_Vector64_AsUInt16: @@ -392,6 +394,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsInt16: case NI_Vector128_AsInt32: case NI_Vector128_AsInt64: + case NI_Vector128_AsNInt: + case NI_Vector128_AsNUInt: case NI_Vector128_AsSByte: case NI_Vector128_AsSingle: case NI_Vector128_AsUInt16: diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index bdc7602fa2bb4..f5c23d159a181 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -24,6 +24,8 @@ HARDWARE_INTRINSIC(Vector64, AsDouble, HARDWARE_INTRINSIC(Vector64, AsInt16, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, AsInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, AsInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector64, AsNInt, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector64, AsNUInt, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, AsSByte, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, AsSingle, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, AsUInt16, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) @@ -125,6 +127,8 @@ HARDWARE_INTRINSIC(Vector128, AsDouble, HARDWARE_INTRINSIC(Vector128, AsInt16, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsNInt, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsNUInt, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsSByte, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsSingle, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsUInt16, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 80fb3d48ef824..e6025f9563ad2 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -36,6 +36,8 @@ HARDWARE_INTRINSIC(Vector128, AsDouble, HARDWARE_INTRINSIC(Vector128, AsInt16, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, AsNInt, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, AsNUInt, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsSByte, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsSingle, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsUInt16, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) @@ -141,6 +143,8 @@ HARDWARE_INTRINSIC(Vector256, AsDouble, HARDWARE_INTRINSIC(Vector256, AsInt16, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsInt32, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsInt64, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, AsNInt, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, AsNUInt, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsSByte, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsSingle, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsUInt16, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 0299efed158c0..0ed59bac1a73d 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -576,6 +576,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_AsInt16: case NI_Vector128_AsInt32: case NI_Vector128_AsInt64: + case NI_Vector128_AsNInt: + case NI_Vector128_AsNUInt: case NI_Vector128_AsSByte: case NI_Vector128_AsSingle: case NI_Vector128_AsUInt16: @@ -588,6 +590,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector256_AsInt16: case NI_Vector256_AsInt32: case NI_Vector256_AsInt64: + case NI_Vector256_AsNInt: + case NI_Vector256_AsNUInt: case NI_Vector256_AsSByte: case NI_Vector256_AsSingle: case NI_Vector256_AsUInt16: From 4f8f773adcf395f1e64fca8f5bad859447ad9b33 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 12 Jan 2023 12:11:42 -0800 Subject: [PATCH 05/10] Ensure Vector*.IsSupported is handled as intrinsic --- src/coreclr/jit/hwintrinsic.cpp | 10 ++- src/coreclr/jit/importercalls.cpp | 72 +++++++++++++++---- src/coreclr/jit/namedintrinsiclist.h | 1 + .../System/Runtime/Intrinsics/Vector128_1.cs | 1 + .../System/Runtime/Intrinsics/Vector256_1.cs | 1 + .../System/Runtime/Intrinsics/Vector512_1.cs | 1 + .../System/Runtime/Intrinsics/Vector64_1.cs | 1 + 7 files changed, 74 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 02e1c80811bf6..341f14a460b42 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -314,7 +314,15 @@ NamedIntrinsic HWIntrinsicInfo::lookupId(Compiler* comp, } #endif - if ((strcmp(methodName, "get_IsSupported") == 0) || isHardwareAcceleratedProp) + bool isSupportedProp = (strcmp(methodName, "get_IsSupported") == 0); + + if (isSupportedProp && (strncmp(className, "Vector", 6) == 0)) + { + // The Vector*.IsSupported props report if T is supported & is specially handled in lookupNamedIntrinsic + return NI_Illegal; + } + + if (isSupportedProp || isHardwareAcceleratedProp) { // The `compSupportsHWIntrinsic` above validates `compSupportsIsa` indicating // that the compiler can emit instructions for the ISA but not whether the diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 51da418d15853..a34ecf4fdffa0 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2357,6 +2357,36 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, return gtNewIconNode(false); } + if (ni == NI_IsSupported_Type) + { + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); + CorInfoType simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + + switch (simdBaseJitType) + { + case CORINFO_TYPE_BYTE: + case CORINFO_TYPE_UBYTE: + case CORINFO_TYPE_SHORT: + case CORINFO_TYPE_USHORT: + case CORINFO_TYPE_INT: + case CORINFO_TYPE_UINT: + case CORINFO_TYPE_LONG: + case CORINFO_TYPE_ULONG: + case CORINFO_TYPE_FLOAT: + case CORINFO_TYPE_DOUBLE: + case CORINFO_TYPE_NATIVEINT: + case CORINFO_TYPE_NATIVEUINT: + { + return gtNewIconNode(true); + } + + default: + { + return gtNewIconNode(false); + } + } + } + if (ni == NI_Throw_PlatformNotSupportedException) { return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); @@ -7326,12 +7356,9 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) result = NI_System_Collections_Generic_Comparer_get_Default; } } - else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(className, "BitOperations") == 0)) + else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(methodName, "PopCount") == 0)) { - if (strcmp(methodName, "PopCount") == 0) - { - result = NI_System_Numerics_BitOperations_PopCount; - } + result = NI_System_Numerics_BitOperations_PopCount; } else if (strcmp(namespaceName, "System.Numerics") == 0) { @@ -7346,11 +7373,16 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) if (result == NI_Illegal) { - if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) - { - // This allows the relevant code paths to be dropped as dead code even - // on platforms where FEATURE_HW_INTRINSICS is not supported. + // This allows the relevant code paths to be dropped as dead code even + // on platforms where FEATURE_HW_INTRINSICS is not supported. + if (strcmp(methodName, "get_IsSupported") == 0) + { + assert(strcmp(className, "Vector`1") == 0); + result = NI_IsSupported_Type; + } + else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) + { result = NI_IsSupported_False; } else if (gtIsRecursiveCall(method)) @@ -7539,11 +7571,27 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) if (result == NI_Illegal) { - if ((strcmp(methodName, "get_IsSupported") == 0) || (strcmp(methodName, "get_IsHardwareAccelerated") == 0)) + // This allows the relevant code paths to be dropped as dead code even + // on platforms where FEATURE_HW_INTRINSICS is not supported. + + if (strcmp(methodName, "get_IsSupported") == 0) { - // This allows the relevant code paths to be dropped as dead code even - // on platforms where FEATURE_HW_INTRINSICS is not supported. + if (strncmp(className, "Vector", 6) == 0) + { + assert((strcmp(className, "Vector64`1") == 0) || + (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || + (strcmp(className, "Vector512`1") == 0)); + result = NI_IsSupported_Type; + } + else + { + result = NI_IsSupported_False; + } + } + else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) + { result = NI_IsSupported_False; } else if (gtIsRecursiveCall(method)) diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index c24b681b9de0d..52d04ffb8b82f 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -114,6 +114,7 @@ enum NamedIntrinsic : unsigned short NI_IsSupported_True, NI_IsSupported_False, NI_IsSupported_Dynamic, + NI_IsSupported_Type, NI_Throw_PlatformNotSupportedException, NI_System_Threading_Interlocked_And, diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs index b8e80993de38d..6f3d3c0faa85e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs @@ -64,6 +64,7 @@ public static int Count /// true if is supported; otherwise, false. public static bool IsSupported { + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] get { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs index 56e19827b31b4..5ccbcadf37607 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs @@ -62,6 +62,7 @@ public static int Count /// true if is supported; otherwise, false. public static bool IsSupported { + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] get { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs index 1edf3894f63d9..e7f824fcba628 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs @@ -62,6 +62,7 @@ public static int Count /// true if is supported; otherwise, false. public static bool IsSupported { + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] get { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs index f2d369bcc9a47..f2ca58318fb29 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs @@ -70,6 +70,7 @@ public static unsafe int Count /// true if is supported; otherwise, false. public static bool IsSupported { + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] get { From 77b9039b124e46ab2198184a62a3330e194484d3 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 13 Jan 2023 11:39:02 -0800 Subject: [PATCH 06/10] Ensure get_Count for Vector64/128/256 and Vector is handled as intrinsic --- src/coreclr/jit/fgbasic.cpp | 17 +++--- src/coreclr/jit/hwintrinsicarm64.cpp | 12 ----- src/coreclr/jit/hwintrinsiclistarm64.h | 2 - src/coreclr/jit/hwintrinsiclistxarch.h | 2 - src/coreclr/jit/hwintrinsicxarch.cpp | 11 ---- src/coreclr/jit/importercalls.cpp | 54 ++++++++++++++++++++ src/coreclr/jit/namedintrinsiclist.h | 1 + src/coreclr/jit/simdashwintrinsic.cpp | 10 ---- src/coreclr/jit/simdashwintrinsiclistarm64.h | 1 - src/coreclr/jit/simdashwintrinsiclistxarch.h | 2 - 10 files changed, 61 insertions(+), 51 deletions(-) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index f5b2ced1b714c..88a1b475a1074 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -1216,25 +1216,20 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed case NI_IsSupported_True: case NI_IsSupported_False: + case NI_IsSupported_Type: { foldableIntrinsic = true; pushedStack.PushConstant(); break; } -#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) - case NI_Vector128_get_Count: - case NI_Vector256_get_Count: - foldableIntrinsic = true; - pushedStack.PushConstant(); - // TODO: check if it's a loop condition - we unroll such loops. - break; -#elif defined(TARGET_ARM64) && defined(FEATURE_HW_INTRINSICS) - case NI_Vector64_get_Count: - case NI_Vector128_get_Count: + + case NI_Vector_GetCount: + { foldableIntrinsic = true; pushedStack.PushConstant(); + // TODO: for FEATURE_SIMD check if it's a loop condition - we unroll such loops. break; -#endif + } default: { diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 8a5e80ca28bdb..579443772d676 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -849,18 +849,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_Vector64_get_Count: - case NI_Vector128_get_Count: - { - assert(!sig->hasThis()); - assert(numArgs == 0); - - GenTreeIntCon* countNode = gtNewIconNode(getSIMDVectorLength(simdSize, simdBaseType), TYP_INT); - countNode->gtFlags |= GTF_ICON_SIMD_COUNT; - retNode = countNode; - break; - } - case NI_Vector64_Divide: case NI_Vector128_Divide: case NI_Vector64_op_Division: diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index f5c23d159a181..45c3815a19361 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -52,7 +52,6 @@ HARDWARE_INTRINSIC(Vector64, EqualsAny, HARDWARE_INTRINSIC(Vector64, ExtractMostSignificantBits, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector64, Floor, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector64, get_AllBitsSet, 8, 0, {INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Vector64, get_Count, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, get_One, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, get_Zero, 8, 0, {INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector64, GetElement, 8, 2, {INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) @@ -160,7 +159,6 @@ HARDWARE_INTRINSIC(Vector128, EqualsAny, HARDWARE_INTRINSIC(Vector128, ExtractMostSignificantBits, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, Floor, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Vector128, get_Count, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi}, HW_Category_Helper, HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, GetElement, 16, 2, {INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index e6025f9563ad2..df526e2eb3c38 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -69,7 +69,6 @@ HARDWARE_INTRINSIC(Vector128, EqualsAny, HARDWARE_INTRINSIC(Vector128, ExtractMostSignificantBits, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, Floor, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_cmpps, INS_cmpps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Vector128, get_Count, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Vector128, GetElement, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_extractps, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) @@ -173,7 +172,6 @@ HARDWARE_INTRINSIC(Vector256, EqualsAny, HARDWARE_INTRINSIC(Vector256, ExtractMostSignificantBits, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector256, Floor, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, get_AllBitsSet, 32, 0, {INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_cmpps, INS_cmpps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector256, get_Count, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, get_One, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector256, get_Zero, 32, 0, {INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, GetElement, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 0ed59bac1a73d..39f2e2cabdf72 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -1397,17 +1397,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_Vector128_get_Count: - case NI_Vector256_get_Count: - { - assert(sig->numArgs == 0); - - GenTreeIntCon* countNode = gtNewIconNode(getSIMDVectorLength(simdSize, simdBaseType), TYP_INT); - countNode->gtFlags |= GTF_ICON_SIMD_COUNT; - retNode = countNode; - break; - } - case NI_Vector128_get_One: case NI_Vector256_get_One: { diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index a34ecf4fdffa0..86e0f18c39c30 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2387,6 +2387,46 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, } } + if (ni == NI_Vector_GetCount) + { + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); + CorInfoType simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + unsigned simdSize = info.compCompHnd->getClassSize(clsHnd); + + switch (simdBaseJitType) + { + case CORINFO_TYPE_BYTE: + case CORINFO_TYPE_UBYTE: + case CORINFO_TYPE_SHORT: + case CORINFO_TYPE_USHORT: + case CORINFO_TYPE_INT: + case CORINFO_TYPE_UINT: + case CORINFO_TYPE_LONG: + case CORINFO_TYPE_ULONG: + case CORINFO_TYPE_FLOAT: + case CORINFO_TYPE_DOUBLE: + case CORINFO_TYPE_NATIVEINT: + case CORINFO_TYPE_NATIVEUINT: + { + var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); + unsigned elementSize = genTypeSize(simdBaseType); + GenTreeIntCon* countNode = gtNewIconNode(simdSize / elementSize, TYP_INT); + +#if defined(FEATURE_SIMD) + countNode->gtFlags |= GTF_ICON_SIMD_COUNT; +#endif // FEATURE_SIMD + + return countNode; + } + + default: + { + ni = NI_Throw_PlatformNotSupportedException; + break; + } + } + } + if (ni == NI_Throw_PlatformNotSupportedException) { return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); @@ -7385,6 +7425,11 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_IsSupported_False; } + else if (strcmp(methodName, "get_Count") == 0) + { + assert(strcmp(className, "Vector`1") == 0); + result = NI_Vector_GetCount; + } else if (gtIsRecursiveCall(method)) { // For the framework itself, any recursive intrinsics will either be @@ -7594,6 +7639,15 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_IsSupported_False; } + else if (strcmp(methodName, "get_Count") == 0) + { + assert((strcmp(className, "Vector64`1") == 0) || + (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || + (strcmp(className, "Vector512`1") == 0)); + + result = NI_Vector_GetCount; + } else if (gtIsRecursiveCall(method)) { // For the framework itself, any recursive intrinsics will either be diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 52d04ffb8b82f..f9c263230b7f3 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -116,6 +116,7 @@ enum NamedIntrinsic : unsigned short NI_IsSupported_Dynamic, NI_IsSupported_Type, NI_Throw_PlatformNotSupportedException, + NI_Vector_GetCount, NI_System_Threading_Interlocked_And, NI_System_Threading_Interlocked_Or, diff --git a/src/coreclr/jit/simdashwintrinsic.cpp b/src/coreclr/jit/simdashwintrinsic.cpp index 5d2dbd7d52ad7..6d36289c126c4 100644 --- a/src/coreclr/jit/simdashwintrinsic.cpp +++ b/src/coreclr/jit/simdashwintrinsic.cpp @@ -737,16 +737,6 @@ GenTree* Compiler::impSimdAsHWIntrinsicSpecial(NamedIntrinsic intrinsic, return gtNewAllBitsSetConNode(retType); } - case NI_VectorT128_get_Count: -#if defined(TARGET_XARCH) - case NI_VectorT256_get_Count: -#endif // TARGET_XARCH - { - GenTreeIntCon* countNode = gtNewIconNode(getSIMDVectorLength(simdSize, simdBaseType), TYP_INT); - countNode->gtFlags |= GTF_ICON_SIMD_COUNT; - return countNode; - } - case NI_Vector2_get_One: case NI_Vector3_get_One: case NI_Vector4_get_One: diff --git a/src/coreclr/jit/simdashwintrinsiclistarm64.h b/src/coreclr/jit/simdashwintrinsiclistarm64.h index 1400eee9d366b..80e37fcaf88ef 100644 --- a/src/coreclr/jit/simdashwintrinsiclistarm64.h +++ b/src/coreclr/jit/simdashwintrinsiclistarm64.h @@ -140,7 +140,6 @@ SIMD_AS_HWINTRINSIC_ID(VectorT128, EqualsAll, SIMD_AS_HWINTRINSIC_ID(VectorT128, EqualsAny, 2, {NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, Floor, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_VectorT128_Floor, NI_VectorT128_Floor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_AllBitsSet, 0, {NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Count, 0, {NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Item, 2, {NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item}, SimdAsHWIntrinsicFlag::InstanceMethod | SimdAsHWIntrinsicFlag::BaseTypeFromThisArg) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_One, 0, {NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Zero, 0, {NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero}, SimdAsHWIntrinsicFlag::None) diff --git a/src/coreclr/jit/simdashwintrinsiclistxarch.h b/src/coreclr/jit/simdashwintrinsiclistxarch.h index f500760f90a1c..24671df67296a 100644 --- a/src/coreclr/jit/simdashwintrinsiclistxarch.h +++ b/src/coreclr/jit/simdashwintrinsiclistxarch.h @@ -140,7 +140,6 @@ SIMD_AS_HWINTRINSIC_ID(VectorT128, EqualsAll, SIMD_AS_HWINTRINSIC_ID(VectorT128, EqualsAny, 2, {NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny, NI_VectorT128_EqualsAny}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, Floor, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_VectorT128_Floor, NI_VectorT128_Floor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_AllBitsSet, 0, {NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet, NI_VectorT128_get_AllBitsSet}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Count, 0, {NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count, NI_VectorT128_get_Count}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Item, 2, {NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item, NI_VectorT128_get_Item}, SimdAsHWIntrinsicFlag::InstanceMethod | SimdAsHWIntrinsicFlag::BaseTypeFromThisArg) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_One, 0, {NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One, NI_VectorT128_get_One}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, get_Zero, 0, {NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero, NI_VectorT128_get_Zero}, SimdAsHWIntrinsicFlag::None) @@ -240,7 +239,6 @@ SIMD_AS_HWINTRINSIC_ID(VectorT256, EqualsAll, SIMD_AS_HWINTRINSIC_ID(VectorT256, EqualsAny, 2, {NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny, NI_VectorT256_EqualsAny}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, Floor, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_VectorT256_Floor, NI_VectorT256_Floor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, get_AllBitsSet, 0, {NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet, NI_VectorT256_get_AllBitsSet}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT256, get_Count, 0, {NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count, NI_VectorT256_get_Count}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, get_Item, 2, {NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item, NI_VectorT256_get_Item}, SimdAsHWIntrinsicFlag::InstanceMethod | SimdAsHWIntrinsicFlag::BaseTypeFromThisArg) SIMD_AS_HWINTRINSIC_ID(VectorT256, get_One, 0, {NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One, NI_VectorT256_get_One}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, get_Zero, 0, {NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero, NI_VectorT256_get_Zero}, SimdAsHWIntrinsicFlag::None) From b2c475212aa785dc48c5d919c9ddd2e18466b88d Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 13 Jan 2023 12:20:43 -0800 Subject: [PATCH 07/10] Apply formatting patch --- src/coreclr/jit/hwintrinsicarm64.cpp | 4 ++-- src/coreclr/jit/hwintrinsicxarch.cpp | 4 ++-- src/coreclr/jit/importercalls.cpp | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 579443772d676..7f651568df0a1 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -467,7 +467,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* idx = gtNewIconNode(2, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, - /* isSimdAsHWIntrinsic */ false); + /* isSimdAsHWIntrinsic */ false); idx = gtNewIconNode(3, TYP_INT); zero = gtNewZeroConNode(TYP_FLOAT); @@ -495,7 +495,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* idx = gtNewIconNode(3, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, - /* isSimdAsHWIntrinsic */ false); + /* isSimdAsHWIntrinsic */ false); break; } diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 39f2e2cabdf72..a253aedc39704 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -679,7 +679,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* idx = gtNewIconNode(2, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, - /* isSimdAsHWIntrinsic */ false); + /* isSimdAsHWIntrinsic */ false); idx = gtNewIconNode(3, TYP_INT); zero = gtNewZeroConNode(TYP_FLOAT); @@ -707,7 +707,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* idx = gtNewIconNode(3, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16, - /* isSimdAsHWIntrinsic */ false); + /* isSimdAsHWIntrinsic */ false); break; } diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 86e0f18c39c30..d6905ec227c72 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7623,10 +7623,8 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { if (strncmp(className, "Vector", 6) == 0) { - assert((strcmp(className, "Vector64`1") == 0) || - (strcmp(className, "Vector128`1") == 0) || - (strcmp(className, "Vector256`1") == 0) || - (strcmp(className, "Vector512`1") == 0)); + assert((strcmp(className, "Vector64`1") == 0) || (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector512`1") == 0)); result = NI_IsSupported_Type; } @@ -7641,10 +7639,8 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) } else if (strcmp(methodName, "get_Count") == 0) { - assert((strcmp(className, "Vector64`1") == 0) || - (strcmp(className, "Vector128`1") == 0) || - (strcmp(className, "Vector256`1") == 0) || - (strcmp(className, "Vector512`1") == 0)); + assert((strcmp(className, "Vector64`1") == 0) || (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector512`1") == 0)); result = NI_Vector_GetCount; } From 4804b8bdef56126e719462a5dcf9bd5342ed22e2 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 13 Jan 2023 13:18:11 -0800 Subject: [PATCH 08/10] Ensure the right simdSize is selected for AsVector128 --- src/coreclr/jit/hwintrinsiclistarm64.h | 2 +- src/coreclr/jit/hwintrinsiclistxarch.h | 2 +- src/coreclr/jit/importercalls.cpp | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 45c3815a19361..627d95cc165c5 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -137,7 +137,7 @@ HARDWARE_INTRINSIC(Vector128, AsVector, HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Vector128, AsVector128, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsVector128, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, BitwiseOr, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index df526e2eb3c38..a31f584354501 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -47,7 +47,7 @@ HARDWARE_INTRINSIC(Vector128, AsVector, HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsdsse2, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector128, AsVector128, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, AsVector128, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, BitwiseOr, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index d6905ec227c72..3495142dae6b7 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7396,9 +7396,12 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) result = NI_System_Collections_Generic_Comparer_get_Default; } } - else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(methodName, "PopCount") == 0)) + else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(className, "BitOperations") == 0)) { - result = NI_System_Numerics_BitOperations_PopCount; + if (strcmp(methodName, "PopCount") == 0) + { + result = NI_System_Numerics_BitOperations_PopCount; + } } else if (strcmp(namespaceName, "System.Numerics") == 0) { From 9f53fd8e3581dbdcd9233c54291d004b28b5603d Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 13 Jan 2023 15:12:47 -0800 Subject: [PATCH 09/10] Ensure AsVector2/3 are correctly handled in lowering --- src/coreclr/jit/hwintrinsiclistarm64.h | 2 +- src/coreclr/jit/hwintrinsiclistxarch.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 627d95cc165c5..b9d1ab59b686d 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -135,7 +135,7 @@ HARDWARE_INTRINSIC(Vector128, AsUInt32, HARDWARE_INTRINSIC(Vector128, AsUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector128, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index a31f584354501..8d5c2d16a35cb 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -44,8 +44,8 @@ HARDWARE_INTRINSIC(Vector128, AsUInt16, HARDWARE_INTRINSIC(Vector128, AsUInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsdsse2, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector128, AsVector2, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsdsse2, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector128, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, BitwiseAnd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) From 2a2258f7f661b125820be7ce9f50584c926afa87 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 13 Jan 2023 18:05:11 -0800 Subject: [PATCH 10/10] Ensure AsVector2 passes in op1 on Arm64 --- src/coreclr/jit/hwintrinsicarm64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 7f651568df0a1..93e36b710e065 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -424,7 +424,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, assert(retType == TYP_SIMD8); op1 = impSIMDPopStack(TYP_SIMD16); - retNode = gtNewSimdHWIntrinsicNode(retType, retNode, NI_Vector128_GetLower, simdBaseJitType, simdSize); + retNode = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_GetLower, simdBaseJitType, simdSize); break; }