diff --git a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp index 553fca5223..127d3fca6e 100644 --- a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp +++ b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,399 @@ typedef struct D3D12_FEATURE_DATA_D3D12_OPTIONS_PREVIEW { #endif +namespace { + +constexpr D3D12_FEATURE LinearAlgebraSupportFeature = + static_cast(77); +constexpr D3D12_FEATURE LinearAlgebraOperationSupportFeature = + static_cast(78); + +struct RuntimeLinearAlgebraTierSupport { + UINT LinearAlgebraTier; +}; + +struct RuntimeMatrixConstructionSupport { + UINT ComponentType; + UINT WaveSize; + UINT MinM; + UINT MinK; + UINT MinN; +}; + +struct RuntimeMatrixMultiplyShape { + UINT M; + UINT K; + UINT N; +}; + +struct RuntimeWaveMatrixMultiplyInputs { + UINT WaveSize; + UINT MatrixAComponentType; + UINT MatrixBComponentType; + UINT AccumulatorComponentType; +}; + +struct RuntimeWaveMatrixMultiplySupport { + RuntimeWaveMatrixMultiplyInputs Inputs; + UINT SupportFlags; + UINT NumShapes; + RuntimeMatrixMultiplyShape *Shapes; +}; + +struct RuntimeThreadGroupMatrixMultiplySupport { + RuntimeWaveMatrixMultiplyInputs WaveInputs; + RuntimeMatrixMultiplyShape Shape; + UINT SupportFlags; + UINT MinThreadGroupSize; + UINT MaxThreadGroupSize; + UINT PreferredThreadGroupSize; +}; + +struct RuntimeThreadVectorMatrixMultiplySupport { + UINT VectorInputType; + UINT MatrixInputType; + UINT BiasInputType; + UINT VectorResultType; + UINT SupportFlags; +}; + +struct RuntimeThreadOuterProductSupport { + UINT InputComponentType; + UINT ResultComponentType; + BOOL Supported; +}; + +struct RuntimeAtomicAccumulateStoreSupport { + UINT ComponentType; + BOOL RWByteAddressBufferSupported; + BOOL GroupSharedSupported; +}; + +struct RuntimeLinearAlgebraOperationSupport { + UINT OperationType; + union { + RuntimeMatrixConstructionSupport MatrixConstruction; + RuntimeWaveMatrixMultiplySupport WaveMatrixMultiply; + RuntimeThreadGroupMatrixMultiplySupport ThreadGroupMatrixMultiply; + RuntimeThreadVectorMatrixMultiplySupport ThreadVectorMatrixMultiply; + RuntimeThreadOuterProductSupport ThreadOuterProduct; + RuntimeAtomicAccumulateStoreSupport AccumulateStore; + }; +}; + +#if defined(DIRECT3D_LINEAR_ALGEBRA) +static_assert(static_cast(D3D12_FEATURE_LINEAR_ALGEBRA_SUPPORT) == + static_cast(LinearAlgebraSupportFeature), + "Linear algebra tier feature ID changed"); +static_assert( + static_cast( + D3D12_FEATURE_LINEAR_ALGEBRA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT) == + static_cast(LinearAlgebraOperationSupportFeature), + "Linear algebra operation feature ID changed"); + +#define ASSERT_RUNTIME_ENUM(RuntimeValue, D3DValue) \ + static_assert(static_cast(RuntimeValue) == \ + static_cast(D3DValue), \ + "Linear algebra runtime enum value changed") + +ASSERT_RUNTIME_ENUM(linalg_test::Tier::NotSupported, + D3D12_LINEAR_ALGEBRA_TIER_NOT_SUPPORTED); +ASSERT_RUNTIME_ENUM(linalg_test::Tier::Tier1_0, D3D12_LINEAR_ALGEBRA_TIER_1_0); +static_assert(static_cast(linalg_test::DataType::None) == 0, + "Linear algebra no-bias datatype value changed"); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::SInt16, + D3D12_LINEAR_ALGEBRA_DATATYPE_SINT16); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::UInt16, + D3D12_LINEAR_ALGEBRA_DATATYPE_UINT16); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::SInt32, + D3D12_LINEAR_ALGEBRA_DATATYPE_SINT32); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::UInt32, + D3D12_LINEAR_ALGEBRA_DATATYPE_UINT32); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::Float16, + D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::Float32, + D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::SInt8, + D3D12_LINEAR_ALGEBRA_DATATYPE_SINT8); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::UInt8, + D3D12_LINEAR_ALGEBRA_DATATYPE_UINT8); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::Float8E4M3FN, + D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT8_E4M3FN); +ASSERT_RUNTIME_ENUM(linalg_test::DataType::Float8E5M2, + D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT8_E5M2); +ASSERT_RUNTIME_ENUM(linalg_test::OperationType::MatrixConstruction, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION); +ASSERT_RUNTIME_ENUM(linalg_test::OperationType::WaveMatrixMultiply, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_WAVE_MATRIX_MULTIPLY); +ASSERT_RUNTIME_ENUM( + linalg_test::OperationType::ThreadGroupMatrixMultiply, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREADGROUP_MATRIX_MULTIPLY); +ASSERT_RUNTIME_ENUM( + linalg_test::OperationType::ThreadVectorMatrixMultiply, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_VECTOR_MATRIX_MULTIPLY); +ASSERT_RUNTIME_ENUM(linalg_test::OperationType::ThreadOuterProduct, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_OUTER_PRODUCT); +ASSERT_RUNTIME_ENUM( + linalg_test::OperationType::AtomicAccumulateStore, + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE); +ASSERT_RUNTIME_ENUM(linalg_test::MultiplicationFlags::None, + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_NONE); +ASSERT_RUNTIME_ENUM(linalg_test::MultiplicationFlags::Supported, + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED); +ASSERT_RUNTIME_ENUM( + linalg_test::MultiplicationFlags::EmulatedInputs, + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS); +ASSERT_RUNTIME_ENUM( + linalg_test::MultiplicationFlags::EmulatedOutputs, + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_OUTPUTS); +ASSERT_RUNTIME_ENUM(linalg_test::MultiplicationFlags::Transpose, + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_TRANSPOSE); + +#undef ASSERT_RUNTIME_ENUM + +#define ASSERT_RUNTIME_ABI(RuntimeType, D3DType) \ + static_assert(sizeof(RuntimeType) == sizeof(D3DType), \ + "Linear algebra runtime ABI size changed"); \ + static_assert(alignof(RuntimeType) == alignof(D3DType), \ + "Linear algebra runtime ABI alignment changed") + +ASSERT_RUNTIME_ABI(RuntimeLinearAlgebraTierSupport, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeMatrixMultiplyShape, + D3D12_LINEAR_ALGEBRA_MATRIX_MULTIPLY_SHAPE); +ASSERT_RUNTIME_ABI(RuntimeWaveMatrixMultiplyInputs, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_INPUTS); +ASSERT_RUNTIME_ABI(RuntimeWaveMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeThreadOuterProductSupport, + D3D12_LINEAR_ALGEBRA_THREAD_OUTER_PRODUCT_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeAtomicAccumulateStoreSupport, + D3D12_LINEAR_ALGEBRA_ATOMIC_ACCUMULATE_STORE_SUPPORT); +ASSERT_RUNTIME_ABI(RuntimeLinearAlgebraOperationSupport, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT); + +#undef ASSERT_RUNTIME_ABI + +#define ASSERT_RUNTIME_OFFSET(RuntimeType, RuntimeField, D3DType, D3DField) \ + static_assert(offsetof(RuntimeType, RuntimeField) == \ + offsetof(D3DType, D3DField), \ + "Linear algebra runtime ABI field offset changed") + +#define ASSERT_RUNTIME_OFFSET_SAME(RuntimeType, D3DType, Field) \ + ASSERT_RUNTIME_OFFSET(RuntimeType, Field, D3DType, Field) + +ASSERT_RUNTIME_OFFSET_SAME(RuntimeLinearAlgebraTierSupport, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_SUPPORT, + LinearAlgebraTier); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT, + ComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT, + WaveSize); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT, + MinM); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT, + MinK); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixConstructionSupport, + D3D12_LINEAR_ALGEBRA_MATRIX_CONSTRUCTION_SUPPORT, + MinN); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixMultiplyShape, + D3D12_LINEAR_ALGEBRA_MATRIX_MULTIPLY_SHAPE, M); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixMultiplyShape, + D3D12_LINEAR_ALGEBRA_MATRIX_MULTIPLY_SHAPE, K); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeMatrixMultiplyShape, + D3D12_LINEAR_ALGEBRA_MATRIX_MULTIPLY_SHAPE, N); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplyInputs, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_INPUTS, + WaveSize); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplyInputs, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_INPUTS, + MatrixAComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplyInputs, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_INPUTS, + MatrixBComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplyInputs, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_INPUTS, + AccumulatorComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_SUPPORT, + Inputs); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_SUPPORT, + SupportFlags); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_SUPPORT, + NumShapes); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeWaveMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_WAVE_MATRIX_MULTIPLY_SUPPORT, + Shapes); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, WaveInputs); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, Shape); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, SupportFlags); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, + MinThreadGroupSize); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, + MaxThreadGroupSize); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadGroupMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREADGROUP_MATRIX_MULTIPLY_SUPPORT, + PreferredThreadGroupSize); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT, + VectorInputType); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT, + MatrixInputType); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT, BiasInputType); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT, + VectorResultType); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeThreadVectorMatrixMultiplySupport, + D3D12_LINEAR_ALGEBRA_THREAD_VECTOR_MATRIX_MULTIPLY_SUPPORT, SupportFlags); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeThreadOuterProductSupport, + D3D12_LINEAR_ALGEBRA_THREAD_OUTER_PRODUCT_SUPPORT, + InputComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeThreadOuterProductSupport, + D3D12_LINEAR_ALGEBRA_THREAD_OUTER_PRODUCT_SUPPORT, + ResultComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeThreadOuterProductSupport, + D3D12_LINEAR_ALGEBRA_THREAD_OUTER_PRODUCT_SUPPORT, + Supported); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeAtomicAccumulateStoreSupport, + D3D12_LINEAR_ALGEBRA_ATOMIC_ACCUMULATE_STORE_SUPPORT, + ComponentType); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeAtomicAccumulateStoreSupport, + D3D12_LINEAR_ALGEBRA_ATOMIC_ACCUMULATE_STORE_SUPPORT, + RWByteAddressBufferSupported); +ASSERT_RUNTIME_OFFSET_SAME(RuntimeAtomicAccumulateStoreSupport, + D3D12_LINEAR_ALGEBRA_ATOMIC_ACCUMULATE_STORE_SUPPORT, + GroupSharedSupported); +ASSERT_RUNTIME_OFFSET_SAME( + RuntimeLinearAlgebraOperationSupport, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, OperationType); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, MatrixConstruction, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + MatrixConstruction); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, WaveMatrixMultiply, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + WaveMatrixMultiply); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, ThreadGroupMatrixMultiply, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + ThreadGroupMatrixMultiply); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, ThreadVectorMatrixMultiply, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + ThreadVectorMatrixMultiply); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, ThreadOuterProduct, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + ThreadOuterProductSupport); +ASSERT_RUNTIME_OFFSET( + RuntimeLinearAlgebraOperationSupport, AccumulateStore, + D3D12_FEATURE_DATA_LINEAR_ALGEBRA_MATRIX_OPERATION_SUPPORT, + AccumulateStore); + +#undef ASSERT_RUNTIME_OFFSET_SAME +#undef ASSERT_RUNTIME_OFFSET +#endif + +constexpr UINT KnownMultiplicationFlags = 0xf; +constexpr UINT MatrixMultiplicationFlags = + static_cast(linalg_test::MultiplicationFlags::Supported) | + static_cast(linalg_test::MultiplicationFlags::EmulatedOutputs); + +bool isValidMultiplicationFlags(UINT Flags, UINT AllowedFlags) { + if ((Flags & ~AllowedFlags) != 0) + return false; + return Flags == 0 || + (Flags & + static_cast(linalg_test::MultiplicationFlags::Supported)) != 0; +} + +bool isFloat8DataType(linalg_test::DataType Type) { + return Type == linalg_test::DataType::Float8E4M3FN || + Type == linalg_test::DataType::Float8E5M2; +} + +bool isValidThreadVectorMultiplicationFlags( + UINT Flags, linalg_test::DataType MatrixInputType) { + if (!isValidMultiplicationFlags(Flags, KnownMultiplicationFlags)) + return false; + const UINT EmulatedInputs = + static_cast(linalg_test::MultiplicationFlags::EmulatedInputs); + return (Flags & EmulatedInputs) == 0 || isFloat8DataType(MatrixInputType); +} + +LPCWSTR dataTypeName(linalg_test::DataType Type) { + using linalg_test::DataType; + switch (Type) { + case DataType::None: + return L"None"; + case DataType::SInt16: + return L"SInt16"; + case DataType::UInt16: + return L"UInt16"; + case DataType::SInt32: + return L"SInt32"; + case DataType::UInt32: + return L"UInt32"; + case DataType::Float16: + return L"Float16"; + case DataType::Float32: + return L"Float32"; + case DataType::SInt8: + return L"SInt8"; + case DataType::UInt8: + return L"UInt8"; + case DataType::Float8E4M3FN: + return L"Float8E4M3FN"; + case DataType::Float8E5M2: + return L"Float8E5M2"; + } + return L"Unknown"; +} + +void setWaveInputs(RuntimeWaveMatrixMultiplyInputs &RuntimeInputs, + const linalg_test::WaveMatrixMultiplyQuery &Query) { + RuntimeInputs.WaveSize = Query.WaveSize; + RuntimeInputs.MatrixAComponentType = + static_cast(Query.MatrixAComponentType); + RuntimeInputs.MatrixBComponentType = + static_cast(Query.MatrixBComponentType); + RuntimeInputs.AccumulatorComponentType = + static_cast(Query.AccumulatorComponentType); +} + +} // namespace + using namespace hlsl_test; static bool useDebugIfaces() { return true; } @@ -619,6 +1013,549 @@ bool isFallbackPathEnabled() { return EnableFallbackValue != 0; } +namespace linalg_test { + +bool MatrixConstructionSupport::valid() const { + const bool AllZero = MinM == 0 && MinK == 0 && MinN == 0; + const bool AllNonZero = MinM != 0 && MinK != 0 && MinN != 0; + return AllZero || AllNonZero; +} + +bool MatrixConstructionSupport::supported() const { + return valid() && MinM != 0; +} + +bool MatrixConstructionSupport::supports(MatrixRole Role, UINT Rows, + UINT Columns) const { + if (!supported()) + return false; + + switch (Role) { + case MatrixRole::A: + return Rows >= MinM && Columns >= MinK; + case MatrixRole::B: + return Rows >= MinK && Columns >= MinN; + case MatrixRole::Accumulator: + return Rows >= MinM && Columns >= MinN; + } + return false; +} + +bool hasFlag(MultiplicationFlags Value, MultiplicationFlags Flag) { + return (static_cast(Value) & static_cast(Flag)) != 0; +} + +bool WaveMatrixMultiplySupport::valid() const { + if (!isValidMultiplicationFlags(static_cast(SupportFlags), + MatrixMultiplicationFlags)) + return false; + if (!hasFlag(SupportFlags, MultiplicationFlags::Supported)) + return Shapes.empty(); + if (Shapes.empty()) + return false; + for (const MatrixMultiplyShape &Shape : Shapes) { + if (Shape.M == 0 || Shape.K == 0 || Shape.N == 0) + return false; + } + return true; +} + +bool WaveMatrixMultiplySupport::supported() const { + return valid() && hasFlag(SupportFlags, MultiplicationFlags::Supported); +} + +bool WaveMatrixMultiplySupport::supportsShape(UINT M, UINT K, UINT N) const { + if (!supported()) + return false; + for (const MatrixMultiplyShape &Shape : Shapes) { + if (Shape.M != 0 && Shape.K != 0 && Shape.N != 0 && M % Shape.M == 0 && + K % Shape.K == 0 && N % Shape.N == 0) + return true; + } + return false; +} + +bool ThreadGroupMatrixMultiplySupport::supported() const { + return valid() && hasFlag(SupportFlags, MultiplicationFlags::Supported); +} + +bool ThreadGroupMatrixMultiplySupport::valid() const { + if (!isValidMultiplicationFlags(static_cast(SupportFlags), + MatrixMultiplicationFlags)) + return false; + if (!hasFlag(SupportFlags, MultiplicationFlags::Supported)) + return true; + return MinThreadGroupSize != 0 && MaxThreadGroupSize >= MinThreadGroupSize && + MaxThreadGroupSize % MinThreadGroupSize == 0 && + (PreferredThreadGroupSize == 0 || + (PreferredThreadGroupSize >= MinThreadGroupSize && + PreferredThreadGroupSize <= MaxThreadGroupSize && + PreferredThreadGroupSize % MinThreadGroupSize == 0)); +} + +bool ThreadGroupMatrixMultiplySupport::supportsThreadGroupSize( + UINT ThreadGroupSize) const { + return supported() && MinThreadGroupSize != 0 && + ThreadGroupSize >= MinThreadGroupSize && + ThreadGroupSize <= MaxThreadGroupSize && + ThreadGroupSize % MinThreadGroupSize == 0; +} + +bool ThreadVectorMatrixMultiplySupport::supported() const { + return valid() && hasFlag(SupportFlags, MultiplicationFlags::Supported); +} + +bool ThreadVectorMatrixMultiplySupport::valid() const { + return isValidThreadVectorMultiplicationFlags(static_cast(SupportFlags), + MatrixInputType); +} + +bool AtomicAccumulateStoreSupport::supports( + AtomicDestination Destination) const { + switch (Destination) { + case AtomicDestination::RWByteAddressBuffer: + return RWByteAddressBufferSupported; + case AtomicDestination::GroupShared: + return GroupSharedSupported; + } + return false; +} + +ScopeFlags legalScopes(OperationType Operation) { + const ScopeFlags Thread = static_cast(ExecutionScope::Thread); + const ScopeFlags Wave = static_cast(ExecutionScope::Wave); + const ScopeFlags ThreadGroup = + static_cast(ExecutionScope::ThreadGroup); + switch (Operation) { + case OperationType::MatrixConstruction: + return Wave | ThreadGroup; + case OperationType::WaveMatrixMultiply: + return Wave; + case OperationType::ThreadGroupMatrixMultiply: + return ThreadGroup; + case OperationType::ThreadVectorMatrixMultiply: + case OperationType::ThreadOuterProduct: + return Thread; + case OperationType::AtomicAccumulateStore: + // The query category spans thread vector/outer-product accumulation and + // Wave/ThreadGroup matrix forms. Individual operations narrow this mask. + return Thread | Wave | ThreadGroup; + } + return 0; +} + +bool isLegalScope(OperationType Operation, ExecutionScope Scope) { + return (legalScopes(Operation) & static_cast(Scope)) != 0; +} + +Applicability classifyApplicability(HRESULT QueryResult, bool Supported, + CapabilityRequirement Requirement) { + if (FAILED(QueryResult)) + return Applicability::Fail; + if (Supported) + return Applicability::Execute; + if (Requirement == CapabilityRequirement::CapabilityGated) + return Applicability::NotApplicable; + return Applicability::Fail; +} + +HRESULT queryTierSupport(ID3D12Device *Device, TierSupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraTierSupport RuntimeSupport = {}; + const HRESULT HR = Device->CheckFeatureSupport( + LinearAlgebraSupportFeature, &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt(L"Linear algebra tier query failed: 0x%08x", HR); + return HR; + } + + if (RuntimeSupport.LinearAlgebraTier != + static_cast(Tier::NotSupported) && + RuntimeSupport.LinearAlgebraTier != static_cast(Tier::Tier1_0)) { + LogCommentFmt(L"Linear algebra tier query returned invalid tier: 0x%x", + RuntimeSupport.LinearAlgebraTier); + return E_UNEXPECTED; + } + + Support.LinearAlgebraTier = + static_cast(RuntimeSupport.LinearAlgebraTier); + LogCommentFmt(L"Linear algebra tier: 0x%x", + static_cast(Support.LinearAlgebraTier)); + return S_OK; +} + +HRESULT queryMatrixConstruction(ID3D12Device *Device, + const MatrixConstructionQuery &Query, + MatrixConstructionSupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::MatrixConstruction); + RuntimeSupport.MatrixConstruction.ComponentType = + static_cast(Query.ComponentType); + RuntimeSupport.MatrixConstruction.WaveSize = Query.WaveSize; + + const HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"MatrixConstruction query failed: type=%s, wave=%u, hr=0x%08x", + dataTypeName(Query.ComponentType), Query.WaveSize, HR); + return HR; + } + + const UINT MinM = RuntimeSupport.MatrixConstruction.MinM; + const UINT MinK = RuntimeSupport.MatrixConstruction.MinK; + const UINT MinN = RuntimeSupport.MatrixConstruction.MinN; + const bool AllZero = MinM == 0 && MinK == 0 && MinN == 0; + const bool AllNonZero = MinM != 0 && MinK != 0 && MinN != 0; + if (!AllZero && !AllNonZero) { + LogCommentFmt( + L"MatrixConstruction query returned malformed minima: type=%s, " + L"wave=%u, MinM=%u, MinK=%u, MinN=%u", + dataTypeName(Query.ComponentType), Query.WaveSize, MinM, MinK, MinN); + return E_UNEXPECTED; + } + + Support.MinM = MinM; + Support.MinK = MinK; + Support.MinN = MinN; + LogCommentFmt( + L"MatrixConstruction query: type=%s, wave=%u, supported=%u, MinM=%u, " + L"MinK=%u, MinN=%u", + dataTypeName(Query.ComponentType), Query.WaveSize, Support.supported(), + Support.MinM, Support.MinK, Support.MinN); + return S_OK; +} + +HRESULT queryWaveMatrixMultiply(ID3D12Device *Device, + const WaveMatrixMultiplyQuery &Query, + WaveMatrixMultiplySupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::WaveMatrixMultiply); + setWaveInputs(RuntimeSupport.WaveMatrixMultiply.Inputs, Query); + + HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"WaveMatrixMultiply query failed: wave=%u, A=%s, B=%s, Acc=%s, " + L"hr=0x%08x", + Query.WaveSize, dataTypeName(Query.MatrixAComponentType), + dataTypeName(Query.MatrixBComponentType), + dataTypeName(Query.AccumulatorComponentType), HR); + return HR; + } + + if (!isValidMultiplicationFlags( + RuntimeSupport.WaveMatrixMultiply.SupportFlags, + MatrixMultiplicationFlags)) { + LogCommentFmt(L"WaveMatrixMultiply query returned invalid flags: 0x%x", + RuntimeSupport.WaveMatrixMultiply.SupportFlags); + return E_UNEXPECTED; + } + + const bool Supported = + (RuntimeSupport.WaveMatrixMultiply.SupportFlags & + static_cast(MultiplicationFlags::Supported)) != 0; + if (!Supported) { + if (RuntimeSupport.WaveMatrixMultiply.NumShapes != 0) { + LogCommentFmt(L"Unsupported WaveMatrixMultiply query returned %u shapes", + RuntimeSupport.WaveMatrixMultiply.NumShapes); + return E_UNEXPECTED; + } + Support.SupportFlags = static_cast( + RuntimeSupport.WaveMatrixMultiply.SupportFlags); + LogCommentFmt(L"WaveMatrixMultiply query: wave=%u, A=%s, B=%s, Acc=%s, " + L"flags=0x%x, shapes=0", + Query.WaveSize, dataTypeName(Query.MatrixAComponentType), + dataTypeName(Query.MatrixBComponentType), + dataTypeName(Query.AccumulatorComponentType), + static_cast(Support.SupportFlags)); + return S_OK; + } + + const UINT RequestedShapes = RuntimeSupport.WaveMatrixMultiply.NumShapes; + if (RequestedShapes == 0) { + LogCommentFmt( + L"Supported WaveMatrixMultiply query returned no native shapes"); + return E_UNEXPECTED; + } + + std::vector RuntimeShapes(RequestedShapes); + RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::WaveMatrixMultiply); + setWaveInputs(RuntimeSupport.WaveMatrixMultiply.Inputs, Query); + RuntimeSupport.WaveMatrixMultiply.NumShapes = RequestedShapes; + RuntimeSupport.WaveMatrixMultiply.Shapes = RuntimeShapes.data(); + + HR = Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"WaveMatrixMultiply shape query failed: wave=%u, A=%s, B=%s, " + L"Acc=%s, requested=%u, hr=0x%08x", + Query.WaveSize, dataTypeName(Query.MatrixAComponentType), + dataTypeName(Query.MatrixBComponentType), + dataTypeName(Query.AccumulatorComponentType), RequestedShapes, HR); + return HR; + } + + if (!isValidMultiplicationFlags( + RuntimeSupport.WaveMatrixMultiply.SupportFlags, + MatrixMultiplicationFlags) || + (RuntimeSupport.WaveMatrixMultiply.SupportFlags & + static_cast(MultiplicationFlags::Supported)) == 0 || + RuntimeSupport.WaveMatrixMultiply.NumShapes == 0 || + RuntimeSupport.WaveMatrixMultiply.NumShapes > RequestedShapes) { + LogCommentFmt( + L"WaveMatrixMultiply shape query returned malformed flags/count: " + L"flags=0x%x, returned=%u, requested=%u", + RuntimeSupport.WaveMatrixMultiply.SupportFlags, + RuntimeSupport.WaveMatrixMultiply.NumShapes, RequestedShapes); + return E_UNEXPECTED; + } + + Support.SupportFlags = static_cast( + RuntimeSupport.WaveMatrixMultiply.SupportFlags); + for (UINT I = 0; I < RuntimeSupport.WaveMatrixMultiply.NumShapes; ++I) { + const RuntimeMatrixMultiplyShape &Shape = RuntimeShapes[I]; + if (Shape.M == 0 || Shape.K == 0 || Shape.N == 0) { + LogCommentFmt( + L"WaveMatrixMultiply query returned zero native shape at index %u: " + L"M=%u, K=%u, N=%u", + I, Shape.M, Shape.K, Shape.N); + return E_UNEXPECTED; + } + Support.Shapes.push_back({Shape.M, Shape.K, Shape.N}); + } + + LogCommentFmt( + L"WaveMatrixMultiply query: wave=%u, A=%s, B=%s, Acc=%s, flags=0x%x, " + L"shapes=%zu", + Query.WaveSize, dataTypeName(Query.MatrixAComponentType), + dataTypeName(Query.MatrixBComponentType), + dataTypeName(Query.AccumulatorComponentType), + static_cast(Support.SupportFlags), Support.Shapes.size()); + for (size_t I = 0; I < Support.Shapes.size(); ++I) { + const MatrixMultiplyShape &Shape = Support.Shapes[I]; + LogCommentFmt(L" Native shape %zu: M=%u, K=%u, N=%u", I, Shape.M, Shape.K, + Shape.N); + } + return S_OK; +} + +HRESULT +queryThreadGroupMatrixMultiply(ID3D12Device *Device, + const ThreadGroupMatrixMultiplyQuery &Query, + ThreadGroupMatrixMultiplySupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::ThreadGroupMatrixMultiply); + setWaveInputs(RuntimeSupport.ThreadGroupMatrixMultiply.WaveInputs, + Query.WaveInputs); + RuntimeSupport.ThreadGroupMatrixMultiply.Shape = { + Query.Shape.M, + Query.Shape.K, + Query.Shape.N, + }; + + const HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"ThreadGroupMatrixMultiply query failed: wave=%u, A=%s, B=%s, " + L"Acc=%s, shape=(%u,%u,%u), hr=0x%08x", + Query.WaveInputs.WaveSize, + dataTypeName(Query.WaveInputs.MatrixAComponentType), + dataTypeName(Query.WaveInputs.MatrixBComponentType), + dataTypeName(Query.WaveInputs.AccumulatorComponentType), Query.Shape.M, + Query.Shape.K, Query.Shape.N, HR); + return HR; + } + + const UINT Flags = RuntimeSupport.ThreadGroupMatrixMultiply.SupportFlags; + if (!isValidMultiplicationFlags(Flags, MatrixMultiplicationFlags)) { + LogCommentFmt( + L"ThreadGroupMatrixMultiply query returned invalid flags: 0x%x", Flags); + return E_UNEXPECTED; + } + + const bool Supported = + (Flags & static_cast(MultiplicationFlags::Supported)) != 0; + const UINT MinSize = + RuntimeSupport.ThreadGroupMatrixMultiply.MinThreadGroupSize; + const UINT MaxSize = + RuntimeSupport.ThreadGroupMatrixMultiply.MaxThreadGroupSize; + const UINT PreferredSize = + RuntimeSupport.ThreadGroupMatrixMultiply.PreferredThreadGroupSize; + if (Supported && + (MinSize == 0 || MaxSize < MinSize || MaxSize % MinSize != 0 || + (PreferredSize != 0 && + (PreferredSize < MinSize || PreferredSize > MaxSize || + PreferredSize % MinSize != 0)))) { + LogCommentFmt( + L"ThreadGroupMatrixMultiply query returned malformed group sizes: " + L"min=%u, max=%u, preferred=%u", + MinSize, MaxSize, PreferredSize); + return E_UNEXPECTED; + } + + Support.SupportFlags = static_cast(Flags); + Support.MinThreadGroupSize = MinSize; + Support.MaxThreadGroupSize = MaxSize; + Support.PreferredThreadGroupSize = PreferredSize; + LogCommentFmt( + L"ThreadGroupMatrixMultiply query: wave=%u, A=%s, B=%s, Acc=%s, " + L"shape=(%u,%u,%u), flags=0x%x, minGroup=%u, maxGroup=%u, " + L"preferredGroup=%u", + Query.WaveInputs.WaveSize, + dataTypeName(Query.WaveInputs.MatrixAComponentType), + dataTypeName(Query.WaveInputs.MatrixBComponentType), + dataTypeName(Query.WaveInputs.AccumulatorComponentType), Query.Shape.M, + Query.Shape.K, Query.Shape.N, Flags, MinSize, MaxSize, PreferredSize); + return S_OK; +} + +HRESULT +queryThreadVectorMatrixMultiply(ID3D12Device *Device, + const ThreadVectorMatrixMultiplyQuery &Query, + ThreadVectorMatrixMultiplySupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::ThreadVectorMatrixMultiply); + RuntimeSupport.ThreadVectorMatrixMultiply.VectorInputType = + static_cast(Query.VectorInputType); + RuntimeSupport.ThreadVectorMatrixMultiply.MatrixInputType = + static_cast(Query.MatrixInputType); + RuntimeSupport.ThreadVectorMatrixMultiply.BiasInputType = + static_cast(Query.BiasInputType); + RuntimeSupport.ThreadVectorMatrixMultiply.VectorResultType = + static_cast(Query.VectorResultType); + + const HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"ThreadVectorMatrixMultiply query failed: vector=%s, matrix=%s, " + L"bias=%s, result=%s, hr=0x%08x", + dataTypeName(Query.VectorInputType), + dataTypeName(Query.MatrixInputType), dataTypeName(Query.BiasInputType), + dataTypeName(Query.VectorResultType), HR); + return HR; + } + + const UINT Flags = RuntimeSupport.ThreadVectorMatrixMultiply.SupportFlags; + if (!isValidThreadVectorMultiplicationFlags(Flags, Query.MatrixInputType)) { + LogCommentFmt( + L"ThreadVectorMatrixMultiply query returned invalid flags: 0x%x", + Flags); + return E_UNEXPECTED; + } + + Support.SupportFlags = static_cast(Flags); + Support.MatrixInputType = Query.MatrixInputType; + LogCommentFmt( + L"ThreadVectorMatrixMultiply query: vector=%s, matrix=%s, bias=%s, " + L"result=%s, flags=0x%x", + dataTypeName(Query.VectorInputType), dataTypeName(Query.MatrixInputType), + dataTypeName(Query.BiasInputType), dataTypeName(Query.VectorResultType), + Flags); + return S_OK; +} + +HRESULT queryThreadOuterProduct(ID3D12Device *Device, + const ThreadOuterProductQuery &Query, + ThreadOuterProductSupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::ThreadOuterProduct); + RuntimeSupport.ThreadOuterProduct.InputComponentType = + static_cast(Query.InputComponentType); + RuntimeSupport.ThreadOuterProduct.ResultComponentType = + static_cast(Query.ResultComponentType); + + const HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt( + L"ThreadOuterProduct query failed: input=%s, result=%s, hr=0x%08x", + dataTypeName(Query.InputComponentType), + dataTypeName(Query.ResultComponentType), HR); + return HR; + } + + Support.Supported = RuntimeSupport.ThreadOuterProduct.Supported != FALSE; + LogCommentFmt(L"ThreadOuterProduct query: input=%s, result=%s, supported=%u", + dataTypeName(Query.InputComponentType), + dataTypeName(Query.ResultComponentType), Support.Supported); + return S_OK; +} + +HRESULT queryAtomicAccumulateStore(ID3D12Device *Device, + const AtomicAccumulateStoreQuery &Query, + AtomicAccumulateStoreSupport &Support) { + Support = {}; + if (!Device) + return E_INVALIDARG; + + RuntimeLinearAlgebraOperationSupport RuntimeSupport = {}; + RuntimeSupport.OperationType = + static_cast(OperationType::AtomicAccumulateStore); + RuntimeSupport.AccumulateStore.ComponentType = + static_cast(Query.ComponentType); + + const HRESULT HR = + Device->CheckFeatureSupport(LinearAlgebraOperationSupportFeature, + &RuntimeSupport, sizeof(RuntimeSupport)); + if (FAILED(HR)) { + LogCommentFmt(L"AtomicAccumulateStore query failed: type=%s, hr=0x%08x", + dataTypeName(Query.ComponentType), HR); + return HR; + } + + Support.RWByteAddressBufferSupported = + RuntimeSupport.AccumulateStore.RWByteAddressBufferSupported != FALSE; + Support.GroupSharedSupported = + RuntimeSupport.AccumulateStore.GroupSharedSupported != FALSE; + LogCommentFmt(L"AtomicAccumulateStore query: type=%s, UAV=%u, groupshared=%u", + dataTypeName(Query.ComponentType), + Support.RWByteAddressBufferSupported, + Support.GroupSharedSupported); + return S_OK; +} + +} // namespace linalg_test + UINT getMaxGroupSharedMemoryCS(ID3D12Device *Device) { D3D12_FEATURE_DATA_D3D12_OPTIONS_PREVIEW O = {}; VERIFY_SUCCEEDED(Device->CheckFeatureSupport( @@ -707,9 +1644,10 @@ void addSRVBuffer(st::ShaderOp *Op, const char *Name, UINT64 Width, Res.Desc.MipLevels = 1; Res.Desc.SampleDesc.Count = 1; Res.Desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; - Res.Desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; + Res.Desc.Flags = D3D12_RESOURCE_FLAG_NONE; Res.InitialResourceState = D3D12_RESOURCE_STATE_COPY_DEST; - Res.TransitionTo = D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; + Res.TransitionTo = D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE | + D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; Op->Resources.push_back(Res); } diff --git a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h index 6cee4e3b8f..c7c6f0db3d 100644 --- a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h +++ b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "ShaderOpTest.h" @@ -74,6 +75,200 @@ bool doesDeviceSupportEnhancedBarriers(ID3D12Device *pDevice); bool doesDeviceSupportRelaxedFormatCasting(ID3D12Device *pDevice); bool isFallbackPathEnabled(); +namespace linalg_test { + +enum class Tier : UINT { + NotSupported = 0, + Tier1_0 = 0x10, +}; + +enum class DataType : UINT { + None = 0, + SInt16 = 2, + UInt16 = 3, + SInt32 = 4, + UInt32 = 5, + Float16 = 7, + Float32 = 8, + SInt8 = 18, + UInt8 = 19, + Float8E4M3FN = 20, + Float8E5M2 = 21, +}; + +enum class OperationType : UINT { + MatrixConstruction = 0, + WaveMatrixMultiply = 1, + ThreadGroupMatrixMultiply = 2, + ThreadVectorMatrixMultiply = 3, + ThreadOuterProduct = 4, + AtomicAccumulateStore = 5, +}; + +enum class MultiplicationFlags : UINT { + None = 0, + Supported = 1, + EmulatedInputs = 2, + EmulatedOutputs = 4, + Transpose = 8, +}; + +enum class MatrixRole { + A, + B, + Accumulator, +}; + +enum class ExecutionScope : UINT { + Thread = 1, + Wave = 2, + ThreadGroup = 4, +}; + +using ScopeFlags = UINT; + +enum class AtomicDestination { + RWByteAddressBuffer, + GroupShared, +}; + +enum class CapabilityRequirement { + Mandatory, + CapabilityGated, +}; + +enum class Applicability { + Execute, + NotApplicable, + Fail, +}; + +struct TierSupport { + Tier LinearAlgebraTier = Tier::NotSupported; + + bool supported() const { return LinearAlgebraTier != Tier::NotSupported; } +}; + +struct MatrixConstructionQuery { + DataType ComponentType; + UINT WaveSize; +}; + +struct MatrixConstructionSupport { + UINT MinM = 0; + UINT MinK = 0; + UINT MinN = 0; + + bool valid() const; + bool supported() const; + bool supports(MatrixRole Role, UINT Rows, UINT Columns) const; +}; + +struct MatrixMultiplyShape { + UINT M; + UINT K; + UINT N; +}; + +struct WaveMatrixMultiplyQuery { + UINT WaveSize; + DataType MatrixAComponentType; + DataType MatrixBComponentType; + DataType AccumulatorComponentType; +}; + +struct WaveMatrixMultiplySupport { + MultiplicationFlags SupportFlags = MultiplicationFlags::None; + std::vector Shapes; + + bool valid() const; + bool supported() const; + bool supportsShape(UINT M, UINT K, UINT N) const; +}; + +struct ThreadGroupMatrixMultiplyQuery { + WaveMatrixMultiplyQuery WaveInputs; + MatrixMultiplyShape Shape; +}; + +struct ThreadGroupMatrixMultiplySupport { + MultiplicationFlags SupportFlags = MultiplicationFlags::None; + UINT MinThreadGroupSize = 0; + UINT MaxThreadGroupSize = 0; + UINT PreferredThreadGroupSize = 0; + + bool valid() const; + bool supported() const; + bool supportsThreadGroupSize(UINT ThreadGroupSize) const; +}; + +struct ThreadVectorMatrixMultiplyQuery { + DataType VectorInputType; + DataType MatrixInputType; + DataType BiasInputType; + DataType VectorResultType; +}; + +struct ThreadVectorMatrixMultiplySupport { + MultiplicationFlags SupportFlags = MultiplicationFlags::None; + DataType MatrixInputType = DataType::None; + + bool valid() const; + bool supported() const; +}; + +struct ThreadOuterProductQuery { + DataType InputComponentType; + DataType ResultComponentType; +}; + +struct ThreadOuterProductSupport { + bool Supported = false; + + bool supported() const { return Supported; } +}; + +struct AtomicAccumulateStoreQuery { + DataType ComponentType; +}; + +struct AtomicAccumulateStoreSupport { + bool RWByteAddressBufferSupported = false; + bool GroupSharedSupported = false; + + bool supports(AtomicDestination Destination) const; +}; + +bool hasFlag(MultiplicationFlags Value, MultiplicationFlags Flag); +ScopeFlags legalScopes(OperationType Operation); +bool isLegalScope(OperationType Operation, ExecutionScope Scope); +Applicability classifyApplicability(HRESULT QueryResult, bool Supported, + CapabilityRequirement Requirement); + +HRESULT queryTierSupport(ID3D12Device *Device, TierSupport &Support); +HRESULT queryMatrixConstruction(ID3D12Device *Device, + const MatrixConstructionQuery &Query, + MatrixConstructionSupport &Support); +HRESULT queryWaveMatrixMultiply(ID3D12Device *Device, + const WaveMatrixMultiplyQuery &Query, + WaveMatrixMultiplySupport &Support); +HRESULT +queryThreadGroupMatrixMultiply(ID3D12Device *Device, + const ThreadGroupMatrixMultiplyQuery &Query, + ThreadGroupMatrixMultiplySupport &Support); +HRESULT +queryThreadVectorMatrixMultiply(ID3D12Device *Device, + const ThreadVectorMatrixMultiplyQuery &Query, + ThreadVectorMatrixMultiplySupport &Support); +HRESULT queryThreadOuterProduct(ID3D12Device *Device, + const ThreadOuterProductQuery &Query, + ThreadOuterProductSupport &Support); +HRESULT queryAtomicAccumulateStore(ID3D12Device *Device, + const AtomicAccumulateStoreQuery &Query, + AtomicAccumulateStoreSupport &Support); + +} // namespace linalg_test + UINT getMaxGroupSharedMemoryCS(ID3D12Device *Device); UINT getMaxGroupSharedMemoryAS(ID3D12Device *Device); UINT getMaxGroupSharedMemoryMS(ID3D12Device *Device); diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index 8a5b0c49f1..eee3736faf 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -25,9 +25,13 @@ #include "HlslTestUtils.h" #include +#include +#include #include #include #include +#include +#include #define STREAM_FLOAT(stream, name, value) \ stream << std::showpoint << " -D" << name << "=" << value << "F" \ @@ -92,6 +96,748 @@ struct MatrixParams { size_t totalBytes() const { return totalElements() * elementSize(CompType); } }; +static std::optional +toCapabilityDataType(ComponentType CompType) { + using linalg_test::DataType; + switch (CompType) { + case ComponentType::I16: + return DataType::SInt16; + case ComponentType::U16: + return DataType::UInt16; + case ComponentType::I32: + return DataType::SInt32; + case ComponentType::U32: + return DataType::UInt32; + case ComponentType::F16: + return DataType::Float16; + case ComponentType::F32: + return DataType::Float32; + case ComponentType::I8: + return DataType::SInt8; + case ComponentType::U8: + return DataType::UInt8; + case ComponentType::F8_E4M3FN: + return DataType::Float8E4M3FN; + case ComponentType::F8_E5M2: + return DataType::Float8E5M2; + default: + return std::nullopt; + } +} + +static linalg_test::ExecutionScope toCapabilityScope(MatrixScope Scope) { + switch (Scope) { + case MatrixScope::Thread: + return linalg_test::ExecutionScope::Thread; + case MatrixScope::Wave: + return linalg_test::ExecutionScope::Wave; + case MatrixScope::ThreadGroup: + return linalg_test::ExecutionScope::ThreadGroup; + } + VERIFY_IS_TRUE(false, "Unsupported LinAlg matrix scope"); + return linalg_test::ExecutionScope::Thread; +} + +static bool applyApplicability(linalg_test::Applicability Result, + LPCWSTR CaseName) { + using linalg_test::Applicability; + switch (Result) { + case Applicability::Execute: + return true; + case Applicability::NotApplicable: + hlsl_test::LogCommentFmt( + L"Capability-gated case %s is not applicable on this device", CaseName); + WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); + return false; + case Applicability::Fail: + hlsl_test::LogErrorFmt(L"Capability evaluation failed for case %s", + CaseName); + VERIFY_IS_TRUE(false, "LinAlg capability evaluation failed"); + return false; + } + VERIFY_IS_TRUE(false, "Unknown LinAlg applicability result"); + return false; +} + +namespace cpu_oracle { + +using TypedMatrixValues = + std::variant, std::vector, + std::vector, std::vector>; + +struct TypedMatrix { + ComponentType CompType; + MatrixDim M; + MatrixDim N; + TypedMatrixValues Values; + + size_t totalElements() const { + return static_cast(M) * static_cast(N); + } +}; + +struct MatrixBufferLayout { + LinalgMatrixLayout Layout; + size_t OffsetBytes; + size_t StrideBytes; +}; + +enum class ComparisonMode { + // Floating-point alternatives allowed by the specification must be listed as + // permitted results; Exact compares the encoded component bits. + Exact, + PermittedResults, + Excluded, +}; + +// MatrixResultOracle models matrix-valued outputs. Operations whose observable +// result is a complete destination buffer need a whole-buffer oracle instead. +struct MatrixResultOracle { + ComparisonMode Mode; + std::vector Candidates; + std::wstring PublicRule; +}; + +template struct NativeComponentTraits { + static constexpr ComponentType CompType = CT; + static constexpr size_t Size = sizeof(T); + + static void store(BYTE *Dest, const T &Value) { + static_assert(std::is_trivially_copyable::value, + "Component must be trivially copyable"); + std::memcpy(Dest, &Value, sizeof(Value)); + } + + static T load(const BYTE *Source) { + T Value; + std::memcpy(&Value, Source, sizeof(Value)); + return Value; + } + + static bool exactMatch(const T &Actual, const T &Expected) { + return std::memcmp(&Actual, &Expected, sizeof(T)) == 0; + } + + static std::wstring format(const T &Value) { + std::wstringstream Stream; + Stream << Value; + return Stream.str(); + } +}; + +template struct ComponentTraits; + +template <> +struct ComponentTraits + : NativeComponentTraits { + static std::wstring format(const float &Value) { + uint32_t Bits; + std::memcpy(&Bits, &Value, sizeof(Bits)); + std::wstringstream Stream; + Stream << Value << L" (bits=0x" << std::hex << Bits << L")"; + return Stream.str(); + } +}; + +template <> +struct ComponentTraits + : NativeComponentTraits {}; + +template <> +struct ComponentTraits + : NativeComponentTraits {}; + +template <> struct ComponentTraits { + static constexpr ComponentType CompType = ComponentType::F16; + static constexpr size_t Size = sizeof(uint16_t); + + static void store(BYTE *Dest, const HLSLHalf_t &Value) { + std::memcpy(Dest, &Value.Val, sizeof(Value.Val)); + } + + static HLSLHalf_t load(const BYTE *Source) { + uint16_t Bits; + std::memcpy(&Bits, Source, sizeof(Bits)); + return HLSLHalf_t::FromHALF(static_cast(Bits)); + } + + static bool exactMatch(const HLSLHalf_t &Actual, const HLSLHalf_t &Expected) { + return Actual.Val == Expected.Val; + } + + static std::wstring format(const HLSLHalf_t &Value) { + std::wstringstream Stream; + Stream << static_cast(Value) << L" (bits=0x" << std::hex << Value.Val + << L")"; + return Stream.str(); + } +}; + +static bool checkedMultiply(size_t Left, size_t Right, size_t &Result) { + if (Right != 0 && Left > std::numeric_limits::max() / Right) + return false; + Result = Left * Right; + return true; +} + +static bool checkedAdd(size_t Left, size_t Right, size_t &Result) { + if (Left > std::numeric_limits::max() - Right) + return false; + Result = Left + Right; + return true; +} + +static bool isSupportedComponentType(ComponentType CompType) { + switch (CompType) { + case ComponentType::F16: + case ComponentType::F32: + case ComponentType::I32: + case ComponentType::U32: + return true; + default: + return false; + } +} + +static LPCWSTR componentTypeName(ComponentType CompType) { + switch (CompType) { + case ComponentType::F16: + return L"F16"; + case ComponentType::F32: + return L"F32"; + case ComponentType::I32: + return L"I32"; + case ComponentType::U32: + return L"U32"; + default: + return L"Unsupported"; + } +} + +static LPCWSTR comparisonModeName(ComparisonMode Mode) { + switch (Mode) { + case ComparisonMode::Exact: + return L"Exact"; + case ComparisonMode::PermittedResults: + return L"PermittedResults"; + case ComparisonMode::Excluded: + return L"Excluded"; + } + return L"Unknown"; +} + +static bool isMatrixValid(const TypedMatrix &Matrix) { + size_t ExpectedElements; + if (Matrix.M == 0 || Matrix.N == 0 || + !checkedMultiply(static_cast(Matrix.M), + static_cast(Matrix.N), ExpectedElements)) + return false; + + switch (Matrix.CompType) { + case ComponentType::F16: + return std::holds_alternative>(Matrix.Values) && + std::get>(Matrix.Values).size() == + ExpectedElements; + case ComponentType::F32: + return std::holds_alternative>(Matrix.Values) && + std::get>(Matrix.Values).size() == + ExpectedElements; + case ComponentType::I32: + return std::holds_alternative>(Matrix.Values) && + std::get>(Matrix.Values).size() == + ExpectedElements; + case ComponentType::U32: + return std::holds_alternative>(Matrix.Values) && + std::get>(Matrix.Values).size() == + ExpectedElements; + default: + return false; + } +} + +template +static std::optional makeTypedMatrix(MatrixDim M, MatrixDim N, + std::vector Values) { + size_t ExpectedElements; + if (M == 0 || N == 0 || + !checkedMultiply(static_cast(M), static_cast(N), + ExpectedElements) || + Values.size() != ExpectedElements) { + hlsl_test::LogErrorFmt( + L"Invalid typed matrix dimensions or element count: M=%u, N=%u, " + L"elements=%zu", + M, N, Values.size()); + return std::nullopt; + } + + return TypedMatrix{ComponentTraits::CompType, M, N, std::move(Values)}; +} + +static std::optional +makeSequentialMatrix(ComponentType CompType, MatrixDim M, MatrixDim N, + uint32_t StartingValue = 1) { + size_t NumElements; + if (M == 0 || N == 0 || + !checkedMultiply(static_cast(M), static_cast(N), + NumElements)) { + hlsl_test::LogErrorFmt(L"Invalid sequential matrix dimensions: M=%u, N=%u", + M, N); + return std::nullopt; + } + + size_t LastValueSize; + if (!checkedAdd(static_cast(StartingValue), NumElements - 1, + LastValueSize)) { + hlsl_test::LogErrorFmt(L"Sequential matrix value calculation overflowed"); + return std::nullopt; + } + const uint64_t LastValue = static_cast(LastValueSize); + + switch (CompType) { + case ComponentType::F16: { + if (LastValue > 65504) { + hlsl_test::LogErrorFmt(L"F16 sequential value is out of range: %llu", + LastValue); + return std::nullopt; + } + std::vector Values; + Values.reserve(NumElements); + for (size_t I = 0; I < NumElements; ++I) + Values.emplace_back(static_cast( + static_cast(StartingValue) + static_cast(I))); + return makeTypedMatrix(M, N, std::move(Values)); + } + case ComponentType::F32: { + if (LastValue > (1u << 24)) { + hlsl_test::LogErrorFmt( + L"F32 sequential integer cannot be represented exactly: %llu", + LastValue); + return std::nullopt; + } + std::vector Values; + Values.reserve(NumElements); + for (size_t I = 0; I < NumElements; ++I) + Values.push_back(static_cast(static_cast(StartingValue) + + static_cast(I))); + return makeTypedMatrix(M, N, std::move(Values)); + } + case ComponentType::I32: { + if (LastValue > + static_cast(std::numeric_limits::max())) { + hlsl_test::LogErrorFmt(L"I32 sequential value is out of range: %llu", + LastValue); + return std::nullopt; + } + std::vector Values; + Values.reserve(NumElements); + for (size_t I = 0; I < NumElements; ++I) + Values.push_back(static_cast( + static_cast(StartingValue) + static_cast(I))); + return makeTypedMatrix(M, N, std::move(Values)); + } + case ComponentType::U32: { + if (LastValue > std::numeric_limits::max()) { + hlsl_test::LogErrorFmt(L"U32 sequential value is out of range: %llu", + LastValue); + return std::nullopt; + } + std::vector Values; + Values.reserve(NumElements); + for (size_t I = 0; I < NumElements; ++I) + Values.push_back(static_cast( + static_cast(StartingValue) + static_cast(I))); + return makeTypedMatrix(M, N, std::move(Values)); + } + default: + hlsl_test::LogErrorFmt(L"Unsupported sequential matrix component type: %u", + static_cast(CompType)); + return std::nullopt; + } +} + +template +static std::optional +transposeTypedMatrix(const TypedMatrix &Source) { + const std::vector &SourceValues = std::get>(Source.Values); + std::vector Result(Source.totalElements()); + for (MatrixDim Row = 0; Row < Source.M; ++Row) { + for (MatrixDim Column = 0; Column < Source.N; ++Column) { + const size_t SourceIndex = static_cast(Row) * Source.N + Column; + const size_t ResultIndex = static_cast(Column) * Source.M + Row; + Result[ResultIndex] = SourceValues[SourceIndex]; + } + } + return makeTypedMatrix(Source.N, Source.M, std::move(Result)); +} + +static std::optional transposeMatrix(const TypedMatrix &Source) { + if (!isMatrixValid(Source)) { + hlsl_test::LogErrorFmt(L"Cannot transpose an invalid typed matrix"); + return std::nullopt; + } + + switch (Source.CompType) { + case ComponentType::F16: + return transposeTypedMatrix(Source); + case ComponentType::F32: + return transposeTypedMatrix(Source); + case ComponentType::I32: + return transposeTypedMatrix(Source); + case ComponentType::U32: + return transposeTypedMatrix(Source); + default: + return std::nullopt; + } +} + +static bool isMemoryLayout(LinalgMatrixLayout Layout) { + return Layout == LinalgMatrixLayout::RowMajor || + Layout == LinalgMatrixLayout::ColumnMajor; +} + +static std::optional +getMatrixBufferSize(ComponentType CompType, MatrixDim M, MatrixDim N, + const MatrixBufferLayout &Layout) { + if (!isSupportedComponentType(CompType) || M == 0 || N == 0 || + !isMemoryLayout(Layout.Layout)) { + hlsl_test::LogErrorFmt( + L"Invalid matrix buffer description: component=%s, M=%u, N=%u, " + L"layout=%u", + componentTypeName(CompType), M, N, + static_cast(Layout.Layout)); + return std::nullopt; + } + + const size_t ElementBytes = elementSize(CompType); + const size_t MajorCount = + Layout.Layout == LinalgMatrixLayout::RowMajor ? M : N; + const size_t MinorCount = + Layout.Layout == LinalgMatrixLayout::RowMajor ? N : M; + size_t PackedMinorBytes; + if (!checkedMultiply(MinorCount, ElementBytes, PackedMinorBytes)) { + hlsl_test::LogErrorFmt( + L"Matrix packed row or column byte calculation overflowed: " + L"component=%s, M=%u, N=%u", + componentTypeName(CompType), M, N); + return std::nullopt; + } + if (Layout.StrideBytes < PackedMinorBytes) { + hlsl_test::LogErrorFmt( + L"Matrix stride is too small: component=%s, M=%u, N=%u, stride=%zu, " + L"required=%zu", + componentTypeName(CompType), M, N, Layout.StrideBytes, + PackedMinorBytes); + return std::nullopt; + } + + size_t LastMajorOffset; + size_t RequiredBytes; + if (!checkedMultiply(MajorCount - 1, Layout.StrideBytes, LastMajorOffset) || + !checkedAdd(Layout.OffsetBytes, LastMajorOffset, RequiredBytes) || + !checkedAdd(RequiredBytes, PackedMinorBytes, RequiredBytes)) { + hlsl_test::LogErrorFmt(L"Matrix buffer size calculation overflowed"); + return std::nullopt; + } + return RequiredBytes; +} + +static std::optional +getMatrixBufferSize(const TypedMatrix &Matrix, + const MatrixBufferLayout &Layout) { + if (!isMatrixValid(Matrix)) { + hlsl_test::LogErrorFmt(L"Cannot size an invalid typed matrix"); + return std::nullopt; + } + return getMatrixBufferSize(Matrix.CompType, Matrix.M, Matrix.N, Layout); +} + +static std::optional +getElementByteOffset(ComponentType CompType, MatrixDim M, MatrixDim N, + MatrixDim Row, MatrixDim Column, + const MatrixBufferLayout &Layout) { + if (Row >= M || Column >= N) + return std::nullopt; + + const size_t Major = + Layout.Layout == LinalgMatrixLayout::RowMajor ? Row : Column; + const size_t Minor = + Layout.Layout == LinalgMatrixLayout::RowMajor ? Column : Row; + size_t MajorOffset; + size_t MinorOffset; + size_t ByteOffset; + if (!checkedMultiply(Major, Layout.StrideBytes, MajorOffset) || + !checkedMultiply(Minor, elementSize(CompType), MinorOffset) || + !checkedAdd(Layout.OffsetBytes, MajorOffset, ByteOffset) || + !checkedAdd(ByteOffset, MinorOffset, ByteOffset)) + return std::nullopt; + return ByteOffset; +} + +template +static bool writeTypedMatrixBuffer(const TypedMatrix &Matrix, + const MatrixBufferLayout &Layout, + std::vector &Buffer) { + const std::vector &Values = std::get>(Matrix.Values); + for (MatrixDim Row = 0; Row < Matrix.M; ++Row) { + for (MatrixDim Column = 0; Column < Matrix.N; ++Column) { + const size_t ValueIndex = static_cast(Row) * Matrix.N + Column; + std::optional ByteOffset = getElementByteOffset( + Matrix.CompType, Matrix.M, Matrix.N, Row, Column, Layout); + if (!ByteOffset.has_value()) + return false; + ComponentTraits::store(Buffer.data() + *ByteOffset, + Values[ValueIndex]); + } + } + return true; +} + +static bool writeMatrixBuffer(const TypedMatrix &Matrix, + const MatrixBufferLayout &Layout, + std::vector &Buffer) { + std::optional RequiredBytes = getMatrixBufferSize(Matrix, Layout); + if (!RequiredBytes.has_value() || Buffer.size() < *RequiredBytes) { + hlsl_test::LogErrorFmt( + L"Matrix buffer is too small: actual=%zu, required=%zu", Buffer.size(), + RequiredBytes.value_or(0)); + return false; + } + + switch (Matrix.CompType) { + case ComponentType::F16: + return writeTypedMatrixBuffer(Matrix, Layout, Buffer); + case ComponentType::F32: + return writeTypedMatrixBuffer(Matrix, Layout, Buffer); + case ComponentType::I32: + return writeTypedMatrixBuffer(Matrix, Layout, Buffer); + case ComponentType::U32: + return writeTypedMatrixBuffer(Matrix, Layout, Buffer); + default: + return false; + } +} + +template +static std::optional +decodeTypedMatrixBuffer(ComponentType CompType, MatrixDim M, MatrixDim N, + const MatrixBufferLayout &Layout, const BYTE *Buffer) { + std::vector Values(static_cast(M) * N); + for (MatrixDim Row = 0; Row < M; ++Row) { + for (MatrixDim Column = 0; Column < N; ++Column) { + const size_t ValueIndex = static_cast(Row) * N + Column; + std::optional ByteOffset = + getElementByteOffset(CompType, M, N, Row, Column, Layout); + if (!ByteOffset.has_value()) + return std::nullopt; + Values[ValueIndex] = ComponentTraits::load(Buffer + *ByteOffset); + } + } + return makeTypedMatrix(M, N, std::move(Values)); +} + +static std::optional +decodeMatrixBuffer(ComponentType CompType, MatrixDim M, MatrixDim N, + const MatrixBufferLayout &Layout, const void *Buffer, + size_t BufferSize) { + std::optional RequiredBytes = + getMatrixBufferSize(CompType, M, N, Layout); + if (!Buffer || !RequiredBytes.has_value() || BufferSize < *RequiredBytes) { + hlsl_test::LogErrorFmt( + L"Cannot decode matrix buffer: actual=%zu, required=%zu", BufferSize, + RequiredBytes.value_or(0)); + return std::nullopt; + } + + const BYTE *Bytes = static_cast(Buffer); + switch (CompType) { + case ComponentType::F16: + return decodeTypedMatrixBuffer(CompType, M, N, Layout, Bytes); + case ComponentType::F32: + return decodeTypedMatrixBuffer(CompType, M, N, Layout, Bytes); + case ComponentType::I32: + return decodeTypedMatrixBuffer(CompType, M, N, Layout, Bytes); + case ComponentType::U32: + return decodeTypedMatrixBuffer(CompType, M, N, Layout, Bytes); + default: + return std::nullopt; + } +} + +template +static bool exactMatrixMatch(const TypedMatrix &Actual, + const TypedMatrix &Expected, + size_t &FirstMismatch) { + const std::vector &ActualValues = std::get>(Actual.Values); + const std::vector &ExpectedValues = + std::get>(Expected.Values); + for (size_t I = 0; I < ActualValues.size(); ++I) { + if (!ComponentTraits::exactMatch(ActualValues[I], ExpectedValues[I])) { + FirstMismatch = I; + return false; + } + } + FirstMismatch = ActualValues.size(); + return true; +} + +static bool exactMatrixMatch(const TypedMatrix &Actual, + const TypedMatrix &Expected, + size_t &FirstMismatch) { + if (!isMatrixValid(Actual) || !isMatrixValid(Expected) || + Actual.CompType != Expected.CompType || Actual.M != Expected.M || + Actual.N != Expected.N) { + FirstMismatch = 0; + return false; + } + + switch (Actual.CompType) { + case ComponentType::F16: + return exactMatrixMatch(Actual, Expected, FirstMismatch); + case ComponentType::F32: + return exactMatrixMatch(Actual, Expected, FirstMismatch); + case ComponentType::I32: + return exactMatrixMatch(Actual, Expected, FirstMismatch); + case ComponentType::U32: + return exactMatrixMatch(Actual, Expected, FirstMismatch); + default: + FirstMismatch = 0; + return false; + } +} + +static std::wstring matrixValueString(const TypedMatrix &Matrix, size_t Index) { + switch (Matrix.CompType) { + case ComponentType::F16: + return ComponentTraits::format( + std::get>(Matrix.Values)[Index]); + case ComponentType::F32: + return ComponentTraits::format( + std::get>(Matrix.Values)[Index]); + case ComponentType::I32: + return ComponentTraits::format( + std::get>(Matrix.Values)[Index]); + case ComponentType::U32: + return ComponentTraits::format( + std::get>(Matrix.Values)[Index]); + default: + return L"unsupported"; + } +} + +static MatrixResultOracle exactResult(TypedMatrix Expected, + std::wstring PublicRule) { + return MatrixResultOracle{ + ComparisonMode::Exact, {std::move(Expected)}, std::move(PublicRule)}; +} + +static MatrixResultOracle permittedResults(std::vector Candidates, + std::wstring PublicRule) { + return MatrixResultOracle{ComparisonMode::PermittedResults, + std::move(Candidates), std::move(PublicRule)}; +} + +static MatrixResultOracle excludedResult(std::wstring PublicRule) { + return MatrixResultOracle{ + ComparisonMode::Excluded, {}, std::move(PublicRule)}; +} + +static bool isOracleValid(const MatrixResultOracle &Oracle) { + if (Oracle.PublicRule.empty()) + return false; + if (Oracle.Mode == ComparisonMode::Excluded) + return Oracle.Candidates.empty(); + if (Oracle.Mode == ComparisonMode::Exact && Oracle.Candidates.size() != 1) + return false; + if (Oracle.Mode == ComparisonMode::PermittedResults && + Oracle.Candidates.size() < 2) + return false; + + const TypedMatrix &First = Oracle.Candidates.front(); + if (!isMatrixValid(First)) + return false; + for (const TypedMatrix &Candidate : Oracle.Candidates) { + if (!isMatrixValid(Candidate) || Candidate.CompType != First.CompType || + Candidate.M != First.M || Candidate.N != First.N) + return false; + } + return true; +} + +static bool +matchesAnyCompleteCandidate(const TypedMatrix &Actual, + const MatrixResultOracle &Oracle, + std::vector *FirstMismatches = nullptr) { + if (!isOracleValid(Oracle) || Oracle.Mode == ComparisonMode::Excluded) + return false; + + if (FirstMismatches) + FirstMismatches->clear(); + for (const TypedMatrix &Candidate : Oracle.Candidates) { + size_t FirstMismatch; + if (exactMatrixMatch(Actual, Candidate, FirstMismatch)) + return true; + if (FirstMismatches) + FirstMismatches->push_back(FirstMismatch); + } + return false; +} + +static bool verifyMatrixBuffer(const void *ActualBuffer, + size_t ActualBufferSize, + const MatrixBufferLayout &Layout, + const MatrixResultOracle &Oracle, bool Verbose) { + if (!isOracleValid(Oracle)) { + hlsl_test::LogErrorFmt(L"Invalid matrix oracle"); + return false; + } + if (Oracle.Mode == ComparisonMode::Excluded) { + hlsl_test::LogErrorFmt( + L"Excluded matrix result cannot be used as a success fallback: %s", + Oracle.PublicRule.c_str()); + return false; + } + + const TypedMatrix &Shape = Oracle.Candidates.front(); + std::optional Actual = decodeMatrixBuffer( + Shape.CompType, Shape.M, Shape.N, Layout, ActualBuffer, ActualBufferSize); + if (!Actual.has_value()) + return false; + + std::vector FirstMismatches; + if (matchesAnyCompleteCandidate(*Actual, Oracle, &FirstMismatches)) { + if (Verbose) { + hlsl_test::LogCommentFmt( + L"Matrix comparison passed: component=%s, M=%u, N=%u, mode=%s, " + L"rule=%s", + componentTypeName(Shape.CompType), Shape.M, Shape.N, + comparisonModeName(Oracle.Mode), Oracle.PublicRule.c_str()); + } + return true; + } + + hlsl_test::LogErrorFmt( + L"No complete matrix candidate matched: component=%s, M=%u, N=%u, " + L"mode=%s, rule=%s", + componentTypeName(Shape.CompType), Shape.M, Shape.N, + comparisonModeName(Oracle.Mode), Oracle.PublicRule.c_str()); + for (size_t CandidateIndex = 0; CandidateIndex < Oracle.Candidates.size(); + ++CandidateIndex) { + const TypedMatrix &Candidate = Oracle.Candidates[CandidateIndex]; + const size_t Mismatch = FirstMismatches[CandidateIndex]; + const size_t Row = Mismatch / Shape.N; + const size_t Column = Mismatch % Shape.N; + hlsl_test::LogErrorFmt( + L"Candidate %zu first mismatch at index=%zu, coordinate=(%zu,%zu): " + L"actual=%s, expected=%s", + CandidateIndex, Mismatch, Row, Column, + matrixValueString(*Actual, Mismatch).c_str(), + matrixValueString(Candidate, Mismatch).c_str()); + } + return false; +} + +} // namespace cpu_oracle + static std::string buildCompilerArgs(const MatrixParams &Params, const char *ExtraDefines = nullptr) { std::stringstream SS; @@ -112,9 +858,15 @@ static std::string buildCompilerArgs(const MatrixParams &Params, case ComponentType::F32: SS << " -DELEM_TYPE=float"; break; - default: + case ComponentType::I32: + SS << " -DELEM_TYPE=int"; + break; + case ComponentType::U32: SS << " -DELEM_TYPE=uint"; break; + default: + VERIFY_IS_TRUE(false, "Unsupported LinAlg component type"); + break; } if (Params.Enable16Bit) SS << " -enable-16bit-types"; @@ -305,6 +1057,289 @@ static VariantCompType makeExpectedVec(ComponentType CompType, false); } +class LinAlgCPUOracleTests { +public: + BEGIN_TEST_CLASS(LinAlgCPUOracleTests) + TEST_METHOD_PROPERTY(L"Priority", L"0") + END_TEST_CLASS() + + TEST_METHOD(TypedMatrixBufferRoundTrip); +}; + +void LinAlgCPUOracleTests::TypedMatrixBufferRoundTrip() { + using namespace cpu_oracle; + + auto VerifyScalarEncoding = [](const std::optional &Matrix, + const std::vector &ExpectedBytes) { + if (!Matrix.has_value()) + return false; + MatrixBufferLayout Layout = { + LinalgMatrixLayout::RowMajor, + /*OffsetBytes=*/0, + /*StrideBytes=*/ExpectedBytes.size(), + }; + std::vector ActualBytes(ExpectedBytes.size(), 0); + MatrixResultOracle Oracle = + exactResult(*Matrix, L"Host scalar encoding and decoding"); + return writeMatrixBuffer(*Matrix, Layout, ActualBytes) && + ActualBytes == ExpectedBytes && + verifyMatrixBuffer(ActualBytes.data(), ActualBytes.size(), Layout, + Oracle, /*Verbose=*/false); + }; + + VERIFY_IS_TRUE(VerifyScalarEncoding( + makeTypedMatrix(1, 1, {HLSLHalf_t(1.5f)}), {0x00, 0x3e})); + VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-2.5f}), + {0x00, 0x00, 0x20, 0xc0})); + VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-7}), + {0xf9, 0xff, 0xff, 0xff})); + VERIFY_IS_TRUE( + VerifyScalarEncoding(makeTypedMatrix(1, 1, {0x89abcdefu}), + {0xef, 0xcd, 0xab, 0x89})); + + const uint32_t AdjacentFloatBits = 0x3f800001; + float AdjacentFloat; + std::memcpy(&AdjacentFloat, &AdjacentFloatBits, sizeof(AdjacentFloat)); + VERIFY_IS_TRUE( + ComponentTraits::format(AdjacentFloat).find(L"3f800001") != + std::wstring::npos); + + std::optional Matrix = + makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); + VERIFY_IS_TRUE(Matrix.has_value()); + + MatrixBufferLayout RowMajor = { + LinalgMatrixLayout::RowMajor, + /*OffsetBytes=*/4, + /*StrideBytes=*/16, + }; + std::optional RowBytes = getMatrixBufferSize(*Matrix, RowMajor); + VERIFY_IS_TRUE(RowBytes.has_value()); + std::vector RowBuffer(*RowBytes, 0xcd); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, RowMajor, RowBuffer)); + const std::vector ExpectedRowBuffer = { + 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + 0x00, 0x03, 0x00, 0x00, 0x00, 0xcd, 0xcd, 0xcd, 0xcd, 0x04, 0x00, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + }; + VERIFY_IS_TRUE(RowBuffer == ExpectedRowBuffer); + MatrixResultOracle Exact = + exactResult(*Matrix, L"Host exact row-major matrix encoding"); + VERIFY_IS_TRUE(verifyMatrixBuffer(RowBuffer.data(), RowBuffer.size(), + RowMajor, Exact, /*Verbose=*/false)); + + MatrixBufferLayout ColumnMajor = { + LinalgMatrixLayout::ColumnMajor, + /*OffsetBytes=*/4, + /*StrideBytes=*/12, + }; + std::optional ColumnBytes = getMatrixBufferSize(*Matrix, ColumnMajor); + VERIFY_IS_TRUE(ColumnBytes.has_value()); + std::vector ColumnBuffer(*ColumnBytes, 0xcd); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, ColumnMajor, ColumnBuffer)); + const std::vector ExpectedColumnBuffer = { + 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xcd, 0xcd, 0xcd, 0xcd, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0xcd, 0xcd, 0xcd, 0xcd, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + }; + VERIFY_IS_TRUE(ColumnBuffer == ExpectedColumnBuffer); + VERIFY_IS_TRUE(verifyMatrixBuffer(ColumnBuffer.data(), ColumnBuffer.size(), + ColumnMajor, Exact, /*Verbose=*/false)); + + std::optional Transposed = transposeMatrix(*Matrix); + std::optional ExpectedTranspose = + makeTypedMatrix(3, 2, {1, 4, 2, 5, 3, 6}); + VERIFY_IS_TRUE(Transposed.has_value()); + VERIFY_IS_TRUE(ExpectedTranspose.has_value()); + size_t FirstMismatch; + VERIFY_IS_TRUE( + exactMatrixMatch(*Transposed, *ExpectedTranspose, FirstMismatch)); + + std::optional MixedActual = + makeTypedMatrix(1, 2, {1, 4}); + std::optional CandidateA = + makeTypedMatrix(1, 2, {1, 2}); + std::optional CandidateB = + makeTypedMatrix(1, 2, {3, 4}); + VERIFY_IS_TRUE(MixedActual.has_value()); + VERIFY_IS_TRUE(CandidateA.has_value()); + VERIFY_IS_TRUE(CandidateB.has_value()); + MatrixResultOracle Permitted = + permittedResults({*CandidateA, *CandidateB}, + L"Host whole-result permitted candidate semantics"); + VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); + Permitted.Candidates.push_back(*MixedActual); + VERIFY_IS_TRUE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); + + MatrixResultOracle Excluded = + excludedResult(L"Host excluded-oracle classification"); + VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*Matrix, Excluded)); + + MatrixParams Params = {}; + Params.M = 2; + Params.N = 3; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = LinalgMatrixLayout::RowMajor; + Params.NumThreads = 4; + Params.CompType = ComponentType::I32; + VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=int") != + std::string::npos); + Params.CompType = ComponentType::U32; + VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=uint") != + std::string::npos); +} + +class LinAlgCapabilityTests { +public: + BEGIN_TEST_CLASS(LinAlgCapabilityTests) + TEST_METHOD_PROPERTY(L"Priority", L"0") + END_TEST_CLASS() + + TEST_METHOD(CapabilityPolicyAndPredicates); +}; + +void LinAlgCapabilityTests::CapabilityPolicyAndPredicates() { + using namespace linalg_test; + + VERIFY_IS_TRUE( + classifyApplicability(S_OK, true, CapabilityRequirement::Mandatory) == + Applicability::Execute); + VERIFY_IS_TRUE(classifyApplicability( + S_OK, false, CapabilityRequirement::CapabilityGated) == + Applicability::NotApplicable); + VERIFY_IS_TRUE( + classifyApplicability(S_OK, false, CapabilityRequirement::Mandatory) == + Applicability::Fail); + VERIFY_IS_TRUE( + classifyApplicability(E_UNEXPECTED, true, + CapabilityRequirement::CapabilityGated) == + Applicability::Fail); + + VERIFY_IS_TRUE( + isLegalScope(OperationType::MatrixConstruction, ExecutionScope::Wave)); + VERIFY_IS_TRUE(isLegalScope(OperationType::MatrixConstruction, + ExecutionScope::ThreadGroup)); + VERIFY_IS_FALSE( + isLegalScope(OperationType::MatrixConstruction, ExecutionScope::Thread)); + VERIFY_IS_TRUE( + isLegalScope(OperationType::WaveMatrixMultiply, ExecutionScope::Wave)); + VERIFY_IS_TRUE(isLegalScope(OperationType::ThreadGroupMatrixMultiply, + ExecutionScope::ThreadGroup)); + VERIFY_IS_TRUE(isLegalScope(OperationType::ThreadVectorMatrixMultiply, + ExecutionScope::Thread)); + VERIFY_IS_TRUE( + isLegalScope(OperationType::ThreadOuterProduct, ExecutionScope::Thread)); + VERIFY_IS_TRUE(isLegalScope(OperationType::AtomicAccumulateStore, + ExecutionScope::Thread)); + VERIFY_IS_TRUE( + isLegalScope(OperationType::AtomicAccumulateStore, ExecutionScope::Wave)); + VERIFY_IS_TRUE(isLegalScope(OperationType::AtomicAccumulateStore, + ExecutionScope::ThreadGroup)); + + MatrixConstructionSupport Construction = {4, 8, 16}; + VERIFY_IS_TRUE(Construction.valid()); + VERIFY_IS_TRUE(Construction.supports(MatrixRole::A, 4, 8)); + VERIFY_IS_TRUE(Construction.supports(MatrixRole::B, 8, 16)); + VERIFY_IS_TRUE(Construction.supports(MatrixRole::Accumulator, 4, 16)); + VERIFY_IS_FALSE(Construction.supports(MatrixRole::A, 3, 8)); + MatrixConstructionSupport InvalidConstruction = {4, 0, 16}; + VERIFY_IS_FALSE(InvalidConstruction.valid()); + + WaveMatrixMultiplySupport Wave = { + MultiplicationFlags::Supported, + {{8, 16, 8}, {16, 16, 16}}, + }; + VERIFY_IS_TRUE(Wave.valid()); + VERIFY_IS_TRUE(Wave.supportsShape(32, 32, 16)); + VERIFY_IS_FALSE(Wave.supportsShape(12, 32, 16)); + WaveMatrixMultiplySupport InvalidWave = { + MultiplicationFlags::EmulatedInputs, + {}, + }; + VERIFY_IS_FALSE(InvalidWave.valid()); + InvalidWave = { + static_cast( + static_cast(MultiplicationFlags::Supported) | + static_cast(MultiplicationFlags::EmulatedInputs)), + {{8, 16, 8}}, + }; + VERIFY_IS_FALSE(InvalidWave.valid()); + + ThreadGroupMatrixMultiplySupport ThreadGroup = { + MultiplicationFlags::Supported, + 32, + 128, + 64, + }; + VERIFY_IS_TRUE(ThreadGroup.valid()); + VERIFY_IS_TRUE(ThreadGroup.supportsThreadGroupSize(64)); + VERIFY_IS_FALSE(ThreadGroup.supportsThreadGroupSize(48)); + ThreadGroup.PreferredThreadGroupSize = 48; + VERIFY_IS_FALSE(ThreadGroup.valid()); + ThreadGroup = { + static_cast( + static_cast(MultiplicationFlags::Supported) | + static_cast(MultiplicationFlags::Transpose)), + 32, + 128, + 64, + }; + VERIFY_IS_FALSE(ThreadGroup.valid()); + + ThreadVectorMatrixMultiplySupport ThreadVector = { + static_cast( + static_cast(MultiplicationFlags::Supported) | + static_cast(MultiplicationFlags::Transpose)), + DataType::Float32, + }; + VERIFY_IS_TRUE(ThreadVector.valid()); + VERIFY_IS_TRUE(ThreadVector.supported()); + ThreadVector.SupportFlags = static_cast( + static_cast(MultiplicationFlags::Supported) | + static_cast(MultiplicationFlags::EmulatedInputs)); + VERIFY_IS_FALSE(ThreadVector.valid()); + ThreadVector.MatrixInputType = DataType::Float8E4M3FN; + VERIFY_IS_TRUE(ThreadVector.valid()); + ThreadVector.SupportFlags = MultiplicationFlags::EmulatedInputs; + VERIFY_IS_FALSE(ThreadVector.valid()); + + ThreadOuterProductSupport OuterProduct = {true}; + VERIFY_IS_TRUE(OuterProduct.supported()); + AtomicAccumulateStoreSupport Atomic = {true, false}; + VERIFY_IS_TRUE(Atomic.supports(AtomicDestination::RWByteAddressBuffer)); + VERIFY_IS_FALSE(Atomic.supports(AtomicDestination::GroupShared)); + + VERIFY_ARE_EQUAL(0u, static_cast(DataType::None)); + MatrixConstructionQuery ConstructionQuery = {DataType::Float32, 32}; + WaveMatrixMultiplyQuery WaveQuery = { + 32, + DataType::Float16, + DataType::Float16, + DataType::Float32, + }; + ThreadGroupMatrixMultiplyQuery ThreadGroupQuery = { + WaveQuery, + {16, 16, 16}, + }; + ThreadVectorMatrixMultiplyQuery ThreadVectorQuery = { + DataType::Float16, + DataType::Float16, + DataType::None, + DataType::Float16, + }; + ThreadOuterProductQuery OuterProductQuery = { + DataType::Float16, + DataType::Float16, + }; + AtomicAccumulateStoreQuery AtomicQuery = {DataType::Float16}; + VERIFY_ARE_EQUAL(32u, ConstructionQuery.WaveSize); + VERIFY_ARE_EQUAL(16u, ThreadGroupQuery.Shape.M); + VERIFY_IS_TRUE(ThreadVectorQuery.BiasInputType == DataType::None); + VERIFY_IS_TRUE(OuterProductQuery.InputComponentType == DataType::Float16); + VERIFY_IS_TRUE(AtomicQuery.ComponentType == DataType::Float16); +} + class DxilConf_SM610_LinAlg { public: BEGIN_TEST_CLASS(DxilConf_SM610_LinAlg) @@ -812,7 +1847,11 @@ static const char CopyConvertShader[] = R"( RWByteAddressBuffer Input : register(u0); RWByteAddressBuffer Output : register(u1); + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else [WaveSize(4, 64)] + #endif [numthreads(NUMTHREADS, 1, 1)] void main() { if (GetGroupWaveIndex() != 0) @@ -833,12 +1872,95 @@ static const char CopyConvertShader[] = R"( } )"; +static HRESULT queryCopyConvertSupport(ID3D12Device *Device, + const MatrixParams &Params, + bool Transpose, bool &Supported, + UINT &SelectedWaveSize) { + Supported = false; + SelectedWaveSize = 0; + if (!Device || Params.Use != MatrixUse::A || + !linalg_test::isLegalScope(linalg_test::OperationType::MatrixConstruction, + toCapabilityScope(Params.Scope))) + return E_INVALIDARG; + + std::optional DataType = + toCapabilityDataType(Params.CompType); + if (!DataType.has_value()) + return E_INVALIDARG; + + linalg_test::TierSupport Tier; + HRESULT HR = linalg_test::queryTierSupport(Device, Tier); + if (FAILED(HR) || !Tier.supported()) + return HR; + + D3D12_FEATURE_DATA_D3D12_OPTIONS1 WaveOptions = {}; + HR = Device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS1, &WaveOptions, + sizeof(WaveOptions)); + if (FAILED(HR)) { + hlsl_test::LogCommentFmt(L"Wave-size capability query failed: 0x%08x", HR); + return HR; + } + if (!WaveOptions.WaveOps) { + hlsl_test::LogCommentFmt( + L"Wave operations are unsupported; MatrixConstruction is not " + L"applicable"); + return S_OK; + } + + const auto IsPowerOfTwo = [](UINT Value) { + return Value != 0 && (Value & (Value - 1)) == 0; + }; + if (!IsPowerOfTwo(WaveOptions.WaveLaneCountMin) || + !IsPowerOfTwo(WaveOptions.WaveLaneCountMax) || + WaveOptions.WaveLaneCountMax < WaveOptions.WaveLaneCountMin) { + hlsl_test::LogCommentFmt( + L"Wave-size capability response is malformed: WaveOps=%u, min=%u, " + L"max=%u", + WaveOptions.WaveOps, WaveOptions.WaveLaneCountMin, + WaveOptions.WaveLaneCountMax); + return E_UNEXPECTED; + } + + MatrixParams Destination = Params; + if (Transpose) { + Destination.M = Params.N; + Destination.N = Params.M; + } + + for (UINT WaveSize = 4; WaveSize <= 64; WaveSize *= 2) { + if (WaveSize < WaveOptions.WaveLaneCountMin || + WaveSize > WaveOptions.WaveLaneCountMax) + continue; + + linalg_test::MatrixConstructionSupport Construction; + HR = linalg_test::queryMatrixConstruction(Device, {*DataType, WaveSize}, + Construction); + if (FAILED(HR)) + return HR; + if (Construction.supports(linalg_test::MatrixRole::A, Params.M, Params.N) && + Construction.supports(linalg_test::MatrixRole::A, Destination.M, + Destination.N)) { + hlsl_test::LogCommentFmt( + L"CopyConvert capability matched wave=%u for source=%ux%u and " + L"destination=%ux%u", + WaveSize, Params.M, Params.N, Destination.M, Destination.N); + Supported = true; + SelectedWaveSize = WaveSize; + return S_OK; + } + } + + hlsl_test::LogCommentFmt( + L"No MatrixConstruction query within shader WaveSize(4,64) supports " + L"CopyConvert source=%ux%u and destination=%ux%u", + Params.M, Params.N, Destination.M, Destination.N); + return S_OK; +} + static void runCopyConvert(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, const MatrixParams &Params, bool Verbose, - bool Transpose) { - const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + bool Transpose, UINT ForcedWaveSize = 0) { MatrixParams DstParams = Params; if (Transpose) { DstParams.M = Params.N; @@ -851,36 +1973,70 @@ static void runCopyConvert(ID3D12Device *Device, ExtraDefs << " -DDST_N_DIM=" << DstParams.N; ExtraDefs << " -DSRC_STRIDE=" << Params.strideBytes(); ExtraDefs << " -DDST_STRIDE=" << DstParams.strideBytes(); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); compileShader(DxcSupport, CopyConvertShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1, - /*Increment=*/true, Transpose); + std::optional Input = + cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Input.has_value(), + "Unable to construct typed CopyConvert input"); + std::optional Expected = + Transpose ? cpu_oracle::transposeMatrix(*Input) : Input; + VERIFY_IS_TRUE(Expected.has_value(), + "Unable to construct independent CopyConvert oracle"); + + cpu_oracle::MatrixBufferLayout SourceLayout = { + Params.Layout, + /*OffsetBytes=*/0, + /*StrideBytes=*/Params.strideBytes(), + }; + cpu_oracle::MatrixBufferLayout DestinationLayout = { + DstParams.Layout, + /*OffsetBytes=*/0, + /*StrideBytes=*/DstParams.strideBytes(), + }; + std::optional SourceBufferSize = + cpu_oracle::getMatrixBufferSize(*Input, SourceLayout); + std::optional DestinationBufferSize = + cpu_oracle::getMatrixBufferSize(*Expected, DestinationLayout); + VERIFY_IS_TRUE(SourceBufferSize.has_value(), + "Unable to size typed CopyConvert input"); + VERIFY_IS_TRUE(DestinationBufferSize.has_value(), + "Unable to size typed CopyConvert output"); + + cpu_oracle::TypedMatrix InputMatrix = *Input; + cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult( + *Expected, + L"HLSL proposal 0035 CopyConvertMatrix transpose and descriptor layout"); // Construct the ShaderOp: two UAV buffers, load from one, store to other. auto Op = createComputeOp(CopyConvertShader, "cs_6_10", "UAV(u0), UAV(u1)", Args.c_str()); - addUAVBuffer(Op.get(), "Input", BufferSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", BufferSize, true); + addUAVBuffer(Op.get(), "Input", *SourceBufferSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", *DestinationBufferSize, true); addRootView(Op.get(), 0, "Input"); addRootView(Op.get(), 1, "Output"); - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [InputMatrix, SourceLayout](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + if (_stricmp(Name, "Input") != 0) + return; + VERIFY_IS_TRUE( + cpu_oracle::writeMatrixBuffer(InputMatrix, SourceLayout, Data), + "Unable to encode typed CopyConvert input"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer( + OutData.data(), OutData.size(), DestinationLayout, Oracle, Verbose)); } void DxilConf_SM610_LinAlg::CopyConvert_Wave_16x16_F16() { @@ -921,9 +2077,23 @@ void DxilConf_SM610_LinAlg::CopyConvert_Wave_4x8_F32_Transpose() { Params.Layout = LinalgMatrixLayout::RowMajor; Params.NumThreads = 64; Params.Enable16Bit = false; + + bool Supported; + UINT SelectedWaveSize; + const HRESULT QueryResult = queryCopyConvertSupport( + D3DDevice, Params, /*Transpose=*/true, Supported, SelectedWaveSize); + const linalg_test::Applicability Applicability = + linalg_test::classifyApplicability( + QueryResult, Supported, + linalg_test::CapabilityRequirement::CapabilityGated); + if (!applyApplicability( + Applicability, + L"CopyConvert_Wave_4x8_F32_Transpose MatrixConstruction")) + return; + // Non-square dimensions make the destination shape and row stride observable. runCopyConvert(D3DDevice, DxcSupport, Params, VerboseLogging, - /*Transpose=*/true); + /*Transpose=*/true, SelectedWaveSize); } static const char MatMatMulShader[] = R"( @@ -1395,7 +2565,6 @@ static D3D12_LINEAR_ALGEBRA_DATATYPE toLinAlgDataType(ComponentType CT) { } static const char OuterProductShader[] = R"( - #define USE_A 0 #define SCOPE_THREAD 0 RWByteAddressBuffer Input : register(u0); @@ -1416,7 +2585,7 @@ static const char OuterProductShader[] = R"( } __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE_THREAD)]] Mat; __builtin_LinAlg_MatrixOuterProduct(Mat, VecA, VecB); @@ -1434,6 +2603,8 @@ static void runOuterProduct(ID3D12Device *Device, VERIFY_IS_TRUE( Params.Layout == LinalgMatrixLayout::OuterProductOptimal, "Outer product must output its matrix in OuterProductOptimal layout"); + VERIFY_IS_TRUE(Params.Use == MatrixUse::Accumulator, + "Outer product must output an accumulator matrix"); const size_t NumVecElements = Params.M + Params.N; const size_t InBuffSize = NumVecElements * elementSize(Params.CompType); const size_t NumMatElements = Params.totalElements(); @@ -1502,6 +2673,7 @@ void DxilConf_SM610_LinAlg::OuterProduct_Thread_16x16_F16() { Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Thread; Params.Layout = LinalgMatrixLayout::OuterProductOptimal; Params.NumThreads = 1; diff --git a/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp b/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp index 62bf7f5155..9ef567fb93 100644 --- a/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp +++ b/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp @@ -587,7 +587,8 @@ void ShaderOpTest::CreatePipelineState() { void ShaderOpTest::CreateResources() { CommandListRefs ResCommandList; - ResCommandList.CreateForDevice(m_pDevice, true); + // Resource initialization may transition to graphics-only states. + ResCommandList.CreateForDevice(m_pDevice, false); ResCommandList.Allocator->SetName( L"ShaderOpTest Resource Creation Allocation"); ResCommandList.Queue->SetName(L"ShaderOpTest Resource Creation Queue");