//===- LLVMSPIRVInternal.h - SPIR-V internal header file -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file declares classes and functions shared by SPIR-V reader/writer. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVINTERNAL_H #define SPIRV_SPIRVINTERNAL_H #include "NameMangleAPI.h" #include "libSPIRV/SPIRVEnum.h" #include "libSPIRV/SPIRVError.h" #include "libSPIRV/SPIRVNameMapEnum.h" #include "libSPIRV/SPIRVType.h" #include "libSPIRV/SPIRVUtil.h" #include "LLVMSPIRVLib.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include #include using namespace SPIRV; using namespace llvm; namespace llvm { class IntrinsicInst; class IRBuilderBase; } namespace SPIRV { /// The LLVM/SPIR-V translator version used to fill the lower 16 bits of the /// generator's magic number in the generated SPIR-V module. /// This number should be bumped up whenever the generated SPIR-V changes. const static unsigned short KTranslatorVer = 14; #define SPCV_TARGET_LLVM_IMAGE_TYPE_ENCODE_ACCESS_QUAL 0 // Workaround for SPIR 2 producer bug about kernel function calling convention. // This workaround checks metadata to determine if a function is kernel. #define SPCV_RELAX_KERNEL_CALLING_CONV 1 class SPIRVOpaqueType; typedef SPIRVMap SPIRVOpaqueTypeOpCodeMap; // Ad hoc function used by LLVM/SPIRV converter for type casting #define SPCV_CAST "spcv.cast" #define LLVM_MEMCPY "llvm.memcpy" // The name of function generated by Clang to initialize sampler(which is // opaqueue type) by 32-bit integer. The name is taken from // CodeGenModule::createOpenCLIntToSamplerConversion(). #define SAMPLER_INIT "__translate_sampler_initializer" template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(Instruction::x, Op##y); /* Casts */ _SPIRV_OP(ZExt, UConvert) _SPIRV_OP(SExt, SConvert) _SPIRV_OP(Trunc, UConvert) _SPIRV_OP(FPToUI, ConvertFToU) _SPIRV_OP(FPToSI, ConvertFToS) _SPIRV_OP(UIToFP, ConvertUToF) _SPIRV_OP(SIToFP, ConvertSToF) _SPIRV_OP(FPTrunc, FConvert) _SPIRV_OP(FPExt, FConvert) _SPIRV_OP(PtrToInt, ConvertPtrToU) _SPIRV_OP(IntToPtr, ConvertUToPtr) _SPIRV_OP(BitCast, Bitcast) _SPIRV_OP(AddrSpaceCast, GenericCastToPtr) _SPIRV_OP(GetElementPtr, AccessChain) _SPIRV_OP(FNeg, FNegate) /*Binary*/ _SPIRV_OP(And, BitwiseAnd) _SPIRV_OP(Or, BitwiseOr) _SPIRV_OP(Xor, BitwiseXor) _SPIRV_OP(Add, IAdd) _SPIRV_OP(FAdd, FAdd) _SPIRV_OP(Sub, ISub) _SPIRV_OP(FSub, FSub) _SPIRV_OP(Mul, IMul) _SPIRV_OP(FMul, FMul) _SPIRV_OP(UDiv, UDiv) _SPIRV_OP(SDiv, SDiv) _SPIRV_OP(FDiv, FDiv) _SPIRV_OP(SRem, SRem) _SPIRV_OP(FRem, FRem) _SPIRV_OP(URem, UMod) _SPIRV_OP(Shl, ShiftLeftLogical) _SPIRV_OP(LShr, ShiftRightLogical) _SPIRV_OP(AShr, ShiftRightArithmetic) #undef _SPIRV_OP } typedef SPIRVMap OpCodeMap; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(CmpInst::x, Op##y); _SPIRV_OP(FCMP_OEQ, FOrdEqual) _SPIRV_OP(FCMP_OGT, FOrdGreaterThan) _SPIRV_OP(FCMP_OGE, FOrdGreaterThanEqual) _SPIRV_OP(FCMP_OLT, FOrdLessThan) _SPIRV_OP(FCMP_OLE, FOrdLessThanEqual) _SPIRV_OP(FCMP_ONE, FOrdNotEqual) _SPIRV_OP(FCMP_ORD, Ordered) _SPIRV_OP(FCMP_UNO, Unordered) _SPIRV_OP(FCMP_UEQ, FUnordEqual) _SPIRV_OP(FCMP_UGT, FUnordGreaterThan) _SPIRV_OP(FCMP_UGE, FUnordGreaterThanEqual) _SPIRV_OP(FCMP_ULT, FUnordLessThan) _SPIRV_OP(FCMP_ULE, FUnordLessThanEqual) _SPIRV_OP(FCMP_UNE, FUnordNotEqual) _SPIRV_OP(ICMP_EQ, IEqual) _SPIRV_OP(ICMP_NE, INotEqual) _SPIRV_OP(ICMP_UGT, UGreaterThan) _SPIRV_OP(ICMP_UGE, UGreaterThanEqual) _SPIRV_OP(ICMP_ULT, ULessThan) _SPIRV_OP(ICMP_ULE, ULessThanEqual) _SPIRV_OP(ICMP_SGT, SGreaterThan) _SPIRV_OP(ICMP_SGE, SGreaterThanEqual) _SPIRV_OP(ICMP_SLT, SLessThan) _SPIRV_OP(ICMP_SLE, SLessThanEqual) #undef _SPIRV_OP } typedef SPIRVMap CmpMap; class IntBoolOpMapId; template <> inline void SPIRVMap::init() { add(OpNot, OpLogicalNot); add(OpBitwiseAnd, OpLogicalAnd); add(OpBitwiseOr, OpLogicalOr); add(OpBitwiseXor, OpLogicalNotEqual); add(OpIEqual, OpLogicalEqual); add(OpINotEqual, OpLogicalNotEqual); } typedef SPIRVMap IntBoolOpMap; #define SPIR_TARGETTRIPLE32 "spir-unknown-unknown" #define SPIR_TARGETTRIPLE64 "spir64-unknown-unknown" #define SPIR_DATALAYOUT32 \ "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" #define SPIR_DATALAYOUT64 \ "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" enum SPIRAddressSpace { SPIRAS_Private, SPIRAS_Global, SPIRAS_Constant, SPIRAS_Local, SPIRAS_Generic, SPIRAS_GlobalDevice, SPIRAS_GlobalHost, SPIRAS_Input, SPIRAS_Output, SPIRAS_Count, }; template <> inline void SPIRVMap::init() { add(SPIRAS_Private, "Private"); add(SPIRAS_Global, "Global"); add(SPIRAS_Constant, "Constant"); add(SPIRAS_Local, "Local"); add(SPIRAS_Generic, "Generic"); add(SPIRAS_Input, "Input"); add(SPIRAS_GlobalDevice, "GlobalDevice"); add(SPIRAS_GlobalHost, "GlobalHost"); } typedef SPIRVMap SPIRAddrSpaceCapitalizedNameMap; template <> inline void SPIRVMap::init() { add(SPIRAS_Private, StorageClassFunction); add(SPIRAS_Global, StorageClassCrossWorkgroup); add(SPIRAS_Constant, StorageClassUniformConstant); add(SPIRAS_Local, StorageClassWorkgroup); add(SPIRAS_Generic, StorageClassGeneric); add(SPIRAS_Input, StorageClassInput); add(SPIRAS_GlobalDevice, StorageClassDeviceOnlyINTEL); add(SPIRAS_GlobalHost, StorageClassHostOnlyINTEL); } typedef SPIRVMap SPIRSPIRVAddrSpaceMap; // Maps OCL builtin function to SPIRV builtin variable. template <> inline void SPIRVMap::init() { add("read_only", AccessQualifierReadOnly); add("write_only", AccessQualifierWriteOnly); add("read_write", AccessQualifierReadWrite); } typedef SPIRVMap SPIRSPIRVAccessQualifierMap; template <> inline void SPIRVMap::init() { add(Attribute::ZExt, FunctionParameterAttributeZext); add(Attribute::SExt, FunctionParameterAttributeSext); add(Attribute::ByVal, FunctionParameterAttributeByVal); add(Attribute::StructRet, FunctionParameterAttributeSret); add(Attribute::NoAlias, FunctionParameterAttributeNoAlias); add(Attribute::NoCapture, FunctionParameterAttributeNoCapture); add(Attribute::ReadOnly, FunctionParameterAttributeNoWrite); add(Attribute::ReadNone, FunctionParameterAttributeNoReadWrite); } typedef SPIRVMap SPIRSPIRVFuncParamAttrMap; template <> inline void SPIRVMap::init() { add(Attribute::ReadNone, FunctionControlPureMask); add(Attribute::ReadOnly, FunctionControlConstMask); add(Attribute::AlwaysInline, FunctionControlInlineMask); add(Attribute::NoInline, FunctionControlDontInlineMask); add(Attribute::OptimizeNone, internal::FunctionControlOptNoneINTELMask); } typedef SPIRVMap SPIRSPIRVFuncCtlMaskMap; class SPIRVExtSetShortName; template <> inline void SPIRVMap::init() { add(SPIRVEIS_OpenCL, "ocl"); } typedef SPIRVMap SPIRVExtSetShortNameMap; template <> inline void SPIRVMap::init() { add(internal::RowMajor, "matrix.rowmajor"); add(internal::ColumnMajor, "matrix.columnmajor"); add(internal::PackedA, "matrix.packed.a"); add(internal::PackedB, "matrix.packed.b"); } typedef SPIRVMap SPIRVMatrixLayoutMap; template <> inline void SPIRVMap::init() { add(ScopeWorkgroup, "scope.workgroup"); add(ScopeSubgroup, "scope.subgroup"); } typedef SPIRVMap SPIRVMatrixScopeMap; #define SPIR_MD_COMPILER_OPTIONS "opencl.compiler.options" #define SPIR_MD_KERNEL_ARG_ADDR_SPACE "kernel_arg_addr_space" #define SPIR_MD_KERNEL_ARG_ACCESS_QUAL "kernel_arg_access_qual" #define SPIR_MD_KERNEL_ARG_TYPE "kernel_arg_type" #define SPIR_MD_KERNEL_ARG_BASE_TYPE "kernel_arg_base_type" #define SPIR_MD_KERNEL_ARG_TYPE_QUAL "kernel_arg_type_qual" #define SPIR_MD_KERNEL_ARG_NAME "kernel_arg_name" #define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations" #define SPIRV_MD_DECORATIONS "spirv.Decorations" #define OCL_TYPE_NAME_SAMPLER_T "sampler_t" #define SPIR_TYPE_NAME_EVENT_T "opencl.event_t" #define SPIR_TYPE_NAME_CLK_EVENT_T "opencl.clk_event_t" #define SPIR_TYPE_NAME_BLOCK_T "opencl.block" #define SPIR_INTRINSIC_BLOCK_BIND "spir_block_bind" #define SPIR_INTRINSIC_GET_BLOCK_INVOKE "spir_get_block_invoke" #define SPIR_INTRINSIC_GET_BLOCK_CONTEXT "spir_get_block_context" #define SPIR_TEMP_NAME_PREFIX_BLOCK "block" #define SPIR_TEMP_NAME_PREFIX_CALL "call" namespace kLLVMTypeName { const static char StructPrefix[] = "struct."; } // namespace kLLVMTypeName namespace kSPIRVImageSampledTypeName { const static char Float[] = "float"; const static char Half[] = "half"; const static char Int[] = "int"; const static char UInt[] = "uint"; const static char Void[] = "void"; } // namespace kSPIRVImageSampledTypeName namespace kSPIRVTypeName { const static char Delimiter = '.'; const static char DeviceEvent[] = "DeviceEvent"; const static char Event[] = "Event"; const static char Image[] = "Image"; const static char Pipe[] = "Pipe"; const static char PostfixDelim = '_'; const static char Prefix[] = "spirv"; const static char PrefixAndDelim[] = "spirv."; const static char Queue[] = "Queue"; const static char ReserveId[] = "ReserveId"; const static char SampledImg[] = "SampledImage"; const static char Sampler[] = "Sampler"; const static char ConstantSampler[] = "ConstantSampler"; const static char PipeStorage[] = "PipeStorage"; const static char ConstantPipeStorage[] = "ConstantPipeStorage"; const static char VmeImageINTEL[] = "VmeImageINTEL"; const static char JointMatrixINTEL[] = "JointMatrixINTEL"; const static char CooperativeMatrixKHR[] = "CooperativeMatrixKHR"; } // namespace kSPIRVTypeName namespace kSPR2TypeName { const static char Delimiter = '.'; const static char OCLPrefix[] = "opencl."; const static char ImagePrefix[] = "opencl.image"; const static char PipeRO[] = "opencl.pipe_ro_t"; const static char PipeWO[] = "opencl.pipe_wo_t"; const static char Sampler[] = "opencl.sampler_t"; const static char Event[] = "opencl.event_t"; } // namespace kSPR2TypeName namespace kAccessQualName { const static char ReadOnly[] = "read_only"; const static char WriteOnly[] = "write_only"; const static char ReadWrite[] = "read_write"; } // namespace kAccessQualName namespace kAccessQualPostfix { const static char ReadOnly[] = "_ro"; const static char WriteOnly[] = "_wo"; const static char ReadWrite[] = "_rw"; const static char Type[] = "_t"; } // namespace kAccessQualPostfix namespace kMangledName { const static char Sampler[] = "11ocl_sampler"; const static char AtomicPrefixIncoming[] = "U7_Atomic"; const static char AtomicPrefixInternal[] = "atomic_"; } // namespace kMangledName namespace kSPIRVName { const static char GroupPrefix[] = "group_"; const static char GroupNonUniformPrefix[] = "group_non_uniform_"; const static char ClusteredPrefix[] = "clustered_"; const static char Prefix[] = "__spirv_"; const static char Postfix[] = "__"; const static char ImageQuerySize[] = "ImageQuerySize"; const static char ImageQuerySizeLod[] = "ImageQuerySizeLod"; const static char ImageSampleExplicitLod[] = "ImageSampleExplicitLod"; const static char ReservedPrefix[] = "reserved_"; const static char SampledImage[] = "SampledImage"; const static char TempSampledImage[] = "TempSampledImage"; const static char TranslateOCLMemOrder[] = "__translate_ocl_memory_order"; const static char TranslateOCLMemScope[] = "__translate_ocl_memory_scope"; const static char TranslateSPIRVMemOrder[] = "__translate_spirv_memory_order"; const static char TranslateSPIRVMemScope[] = "__translate_spirv_memory_scope"; const static char TranslateSPIRVMemFence[] = "__translate_spirv_memory_fence"; const static char EntrypointPrefix[] = "__spirv_entry_"; } // namespace kSPIRVName namespace kSPIRVPostfix { const static char ToGlobal[] = "ToGlobal"; const static char ToLocal[] = "ToLocal"; const static char ToPrivate[] = "ToPrivate"; const static char Sat[] = "sat"; const static char Rtz[] = "rtz"; const static char Rte[] = "rte"; const static char Rtp[] = "rtp"; const static char Rtn[] = "rtn"; const static char Rt[] = "rt"; const static char Return[] = "R"; const static char Divider[] = "_"; /// Divider between extended instruction name and postfix const static char ExtDivider[] = "__"; } // namespace kSPIRVPostfix namespace kSPIRVMD { const static char Capability[] = "spirv.Capability"; const static char EntryPoint[] = "spirv.EntryPoint"; const static char ExecutionMode[] = "spirv.ExecutionMode"; const static char Extension[] = "spirv.Extension"; const static char Generator[] = "spirv.Generator"; const static char Source[] = "spirv.Source"; const static char SourceExtension[] = "spirv.SourceExtension"; const static char MemoryModel[] = "spirv.MemoryModel"; } // namespace kSPIRVMD namespace kSPIR2MD { const static char Extensions[] = "opencl.used.extensions"; const static char FPContract[] = "opencl.enable.FP_CONTRACT"; const static char OCLVer[] = "opencl.ocl.version"; const static char OptFeatures[] = "opencl.used.optional.core.features"; const static char SPIRVer[] = "opencl.spir.version"; const static char VecTyHint[] = "vec_type_hint"; const static char WGSize[] = "reqd_work_group_size"; const static char WGSizeHint[] = "work_group_size_hint"; const static char SubgroupSize[] = "intel_reqd_sub_group_size"; const static char MaxWGSize[] = "max_work_group_size"; const static char NoGlobalOffset[] = "no_global_work_offset"; const static char MaxWGDim[] = "max_global_work_dim"; const static char NumSIMD[] = "num_simd_work_items"; const static char StallEnable[] = "stall_enable"; const static char FmaxMhz[] = "scheduler_target_fmax_mhz"; const static char LoopFuse[] = "loop_fuse"; const static char PreferDSP[] = "prefer_dsp"; const static char PropDSPPref[] = "propagate_dsp_preference"; const static char InitiationInterval[] = "initiation_interval"; const static char MaxConcurrency[] = "max_concurrency"; const static char DisableLoopPipelining[] = "disable_loop_pipelining"; const static char IntelFPGAIPInterface[] = "ip_interface"; } // namespace kSPIR2MD enum Spir2SamplerKind { CLK_ADDRESS_NONE = 0x0000, CLK_ADDRESS_CLAMP = 0x0004, CLK_ADDRESS_CLAMP_TO_EDGE = 0x0002, CLK_ADDRESS_REPEAT = 0x0006, CLK_ADDRESS_MIRRORED_REPEAT = 0x0008, CLK_NORMALIZED_COORDS_FALSE = 0x0000, CLK_NORMALIZED_COORDS_TRUE = 0x0001, CLK_FILTER_NEAREST = 0x0010, CLK_FILTER_LINEAR = 0x0020, }; /// Additional information for mangling a function argument type. struct BuiltinArgTypeMangleInfo { bool IsSigned; bool IsVoidPtr; bool IsEnum; bool IsSampler; bool IsAtomic; bool IsLocalArgBlock; SPIR::TypePrimitiveEnum Enum; unsigned Attr; PointerIndirectPair PointerElementType; BuiltinArgTypeMangleInfo() : IsSigned(true), IsVoidPtr(false), IsEnum(false), IsSampler(false), IsAtomic(false), IsLocalArgBlock(false), Enum(SPIR::PRIMITIVE_NONE), Attr(0), PointerElementType(nullptr, false) {} }; /// Information for mangling builtin function. class BuiltinFuncMangleInfo { public: /// Translate builtin function name and set /// argument attributes and unsigned args. BuiltinFuncMangleInfo(const std::string &UniqName = "") : VarArgIdx(-1), DontMangle(false) { if (!UniqName.empty()) init(UniqName); } virtual ~BuiltinFuncMangleInfo() {} const std::string &getUnmangledName() const { return UnmangledName; } void addUnsignedArg(int Ndx) { if (Ndx == -1) return addUnsignedArgs(0, 10); // 10 is enough for everybody, right? getTypeMangleInfo(Ndx).IsSigned = false; } void addUnsignedArgs(int StartNdx, int StopNdx) { assert(StartNdx < StopNdx && "wrong parameters"); for (int I = StartNdx; I <= StopNdx; ++I) addUnsignedArg(I); } void addVoidPtrArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsVoidPtr = true; } void addSamplerArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsSampler = true; } void addAtomicArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsAtomic = true; } void setLocalArgBlock(unsigned Ndx) { getTypeMangleInfo(Ndx).IsLocalArgBlock = true; } void setEnumArg(unsigned Ndx, SPIR::TypePrimitiveEnum Enum) { auto &Info = getTypeMangleInfo(Ndx); Info.IsEnum = true; Info.Enum = Enum; } void setArgAttr(unsigned Ndx, unsigned Attr) { getTypeMangleInfo(Ndx).Attr = Attr; } void setVarArg(int Ndx) { assert(0 <= Ndx && "it is not allowed to set less than zero index"); VarArgIdx = Ndx; } void setAsDontMangle() { DontMangle = true; } bool avoidMangling() { return DontMangle; } // get ellipsis index, single ellipsis at the end of the function is possible // only return value < 0 if none int getVarArg() const { return VarArgIdx; } BuiltinArgTypeMangleInfo &getTypeMangleInfo(unsigned Ndx) { while (Ndx >= ArgInfo.size()) ArgInfo.emplace_back(); BuiltinArgTypeMangleInfo &Info = ArgInfo[Ndx]; return Info; } virtual void init(StringRef UniqUnmangledName) { UnmangledName = UniqUnmangledName.str(); } void fillPointerElementTypes(ArrayRef); protected: std::string UnmangledName; std::vector ArgInfo; int VarArgIdx; // index of ellipsis argument, idx < 0 if none private: bool DontMangle; // clang doesn't apply mangling for some builtin functions // (i.e. enqueue_kernel) }; /// \returns a vector of types for a collection of values. template std::vector getTypes(T V) { std::vector Tys; for (auto &I : V) Tys.push_back(I->getType()); return Tys; } /// Move elements of std::vector from [begin, end) to target. template void move(std::vector &V, size_t Begin, size_t End, size_t Target) { assert(Begin < End && End <= V.size() && Target <= V.size() && !(Begin < Target && Target < End)); if (Begin <= Target && Target <= End) return; auto B = V.begin() + Begin, E = V.begin() + End; if (Target > V.size()) Target = V.size(); if (Target > End) Target -= (End - Begin); std::vector Segment(B, E); V.erase(B, E); V.insert(V.begin() + Target, Segment.begin(), Segment.end()); } /// Find position of first pointer type value in a vector. inline size_t findFirstPtr(const std::vector &Args) { auto PtArg = std::find_if(Args.begin(), Args.end(), [](Value *V) { return V->getType()->isPointerTy(); }); return PtArg - Args.begin(); } bool isSupportedTriple(Triple T); void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr); void addFnAttr(CallInst *Call, Attribute::AttrKind Attr); void saveLLVMModule(Module *M, const std::string &OutputFile); std::string mapSPIRVTypeToOCLType(SPIRVType *Ty, bool Signed); std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed, Type *PointerElementType = nullptr); SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target); /// Return vector V extended with poison elements to match the number of /// components of NewType. Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder); /// Add decorations to a SPIR-V entry. /// \param Decs Each string is a postfix without _ at the beginning. SPIRVValue *addDecorations(SPIRVValue *Target, const SmallVectorImpl &Decs); PointerType *getOrCreateOpaquePtrType(Module *M, const std::string &Name, unsigned AddrSpace = SPIRAS_Global); StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name); PointerType *getSamplerType(Module *M); Type *getSamplerStructType(Module *M); PointerType *getSPIRVOpaquePtrType(Module *M, Op OC); void getFunctionTypeParameterTypes(llvm::FunctionType *FT, std::vector &ArgTys); Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef ArgTypes, StringRef Name, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); PointerType *getOCLClkEventType(Module *M); PointerType *getOCLClkEventPtrType(Module *M); Constant *getOCLNullClkEventPtr(Module *M); /// Get function call arguments. /// \param Start Starting index. /// \param End Ending index. std::vector getArguments(CallInst *CI, unsigned Start = 0, unsigned End = 0); /// Get constant function call argument as an integer. /// \param I argument index. uint64_t getArgAsInt(CallInst *CI, unsigned I); /// Get constant function call argument as type \param T. /// \param I argument index. template T getArgAs(CallInst *CI, unsigned I) { return static_cast(getArgAsInt(CI, I)); } /// Get constant function call argument as a Scope enum. /// \param I argument index. Scope getArgAsScope(CallInst *CI, unsigned I); /// Get constant function call argument as a Decoration enum. /// \param I argument index. Decoration getArgAsDecoration(CallInst *CI, unsigned I); /// Check if a type is SPIRV sampler type. bool isSPIRVSamplerType(llvm::Type *Ty); /// Check if a type is OCL image type (if pointed to). /// \return type name without "opencl." prefix. bool isOCLImageStructType(llvm::Type *Ty, StringRef *Name = nullptr); /// \param BaseTyName is the type name as in spirv.BaseTyName.Postfixes /// \param Postfix contains postfixes extracted from the SPIR-V image /// type name as spirv.BaseTyName.Postfixes. bool isSPIRVStructType(llvm::Type *Ty, StringRef BaseTyName, StringRef *Postfix = 0); bool isSYCLHalfType(llvm::Type *Ty); bool isSYCLBfloat16Type(llvm::Type *Ty); /// Decorate a function name as __spirv_{Name}_ std::string decorateSPIRVFunction(const std::string &S); /// Remove prefix/postfix from __spirv_{Name}_ StringRef undecorateSPIRVFunction(StringRef S); /// Check if a function has decorated name as __spirv_{Name}_ /// and get the original name. bool isDecoratedSPIRVFunc(const Function *F, StringRef &UndecName); std::string prefixSPIRVName(const std::string &S); StringRef dePrefixSPIRVName(StringRef R, SmallVectorImpl &Postfix); /// Get a canonical function name for a SPIR-V op code. std::string getSPIRVFuncName(Op OC, StringRef PostFix = ""); std::string getSPIRVFuncName(Op OC, const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); std::string getSPIRVFuncName(SPIRVBuiltinVariableKind BVKind); /// Get a canonical function name for a SPIR-V extended instruction std::string getSPIRVExtFuncName(SPIRVExtInstSetKind Set, unsigned ExtOp, StringRef PostFix = ""); /// Get SPIR-V op code given the canonical function name. /// Assume \param Name is either IA64 mangled or unmangled, and the unmangled /// name takes the __spirv_{OpName}_{Postfixes} format. /// \return op code if the unmangled function name is a valid op code name, /// otherwise return OpNop. /// \param Dec contains decorations decoded from function name if it is /// not nullptr. Op getSPIRVFuncOC(StringRef Name, SmallVectorImpl *Dec = nullptr); /// Get SPIR-V builtin variable enum given the canonical builtin name /// Assume \param Name is in format __spirv_BuiltIn{Name} /// \return false if \param Name is not a valid builtin name. bool getSPIRVBuiltin(const std::string &Name, spv::BuiltIn &Builtin); /// \param Name LLVM function name /// \param DemangledName demanged name of the OpenCL built-in function /// \returns true if Name is the name of the OpenCL built-in function, /// false for other functions bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp = false); /// Check if a function returns void bool isVoidFuncTy(FunctionType *FT); /// \returns true if function \p F has array type argument. bool hasArrayArg(Function *F); /// Mutates function call instruction by changing the arguments. /// \param ArgMutate mutates the function arguments. /// \return mutated call instruction. CallInst *mutateCallInst( Module *M, CallInst *CI, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutates function call instruction by changing the arguments and return /// value. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the return value. /// \return mutated instruction. Instruction *mutateCallInst( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutate call instruction to call SPIR-V builtin function. CallInst *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &)> ArgMutate, AttributeList *Attrs = nullptr); /// Mutate call instruction to call SPIR-V builtin function. Instruction *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, AttributeList *Attrs = nullptr); /// Mutate function by change the arguments. /// \param ArgMutate mutates the function arguments. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Mutate function by change the arguments & the return type. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the function return value. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Add a call instruction at \p Pos. CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, Instruction *Pos, BuiltinFuncMangleInfo *Mangle = nullptr, StringRef InstName = SPIR_TEMP_NAME_PREFIX_CALL, bool TakeFuncName = true); /// Add a call instruction for SPIR-V builtin function. CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, ArrayRef PointerElementTypes, Instruction *Pos, StringRef InstName); /// Add a call of spir_block_bind function. CallInst *addBlockBind(Module *M, Function *InvokeFunc, Value *BlkCtx, Value *CtxLen, Value *CtxAlign, Instruction *InsPos, StringRef InstName = SPIR_TEMP_NAME_PREFIX_BLOCK); typedef std::pair::iterator, std::vector::iterator> ValueVecRange; /// Add a vector at \param InsPos. Value *addVector(Instruction *InsPos, ValueVecRange Range); /// Replace scalar values with a vector created at \param InsPos. void makeVector(Instruction *InsPos, std::vector &Ops, ValueVecRange Range); /// Expand a vector type value in \param Ops at index \param VecPos. /// Generate extract element instructions at \param InsPos and replace /// the vector type value with scalar type values. /// If the value to be expanded is not vector type, do nothing. void expandVector(Instruction *InsPos, std::vector &Ops, size_t VecPos); /// Get size_t type. IntegerType *getSizetType(Module *M); /// Get void(void) function type. Type *getVoidFuncType(Module *M); /// Get void(void) function pointer type. Type *getVoidFuncPtrType(Module *M, unsigned AddrSpace = 0); /// Get a 64 bit integer constant. ConstantInt *getInt64(Module *M, int64_t Value); /// Get a 32 bit integer constant. ConstantInt *getInt32(Module *M, int Value); /// Get a 32 bit unsigned integer constant. ConstantInt *getUInt32(Module *M, unsigned Value); /// Get 32 bit integer constant if the value fits in 32 bits, /// return 64 bit integer constant otherwise ConstantInt *getInt(Module *M, int64_t Value); /// Get 32 bit unsigned integer constant if the value fits in 32 bits, /// return 64 bit unsigned integer constant otherwise ConstantInt *getUInt(Module *M, uint64_t Value); /// Get a 16 bit unsigned integer constant. ConstantInt *getUInt16(Module *M, unsigned short Value); // Get a 32 bit floating point constant. Constant *getFloat32(Module *M, float Value); /// Get a 32 bit integer constant vector. std::vector getInt32(Module *M, const std::vector &Value); /// Get a size_t type constant. ConstantInt *getSizet(Module *M, uint64_t Value); /// Get metadata operand as int. int64_t getMDOperandAsInt(MDNode *N, unsigned I); /// Get metadata operand as string. StringRef getMDOperandAsString(MDNode *N, unsigned I); /// Get metadata operand as another metadata node MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I); /// Get metadata operand as type. Type *getMDOperandAsType(MDNode *N, unsigned I); /// Get a named metadata as a set of string. /// Assume the named metadata has one or more operands each of which might /// contain set of strings. For instance: /// !opencl.used.optional.core.features = !{!0} /// !0 = !{!"cl_doubles", !"cl_images"} /// or if we linked two modules we may have /// !opencl.used.optional.core.features = !{!0, !1} /// !0 = !{!"cl_doubles"} /// !1 = !{!"cl_images"} std::set getNamedMDAsStringSet(Module *M, const std::string &MDName); /// Get SPIR-V language by SPIR-V metadata spirv.Source std::tuple getSPIRVSource(Module *M); /// Map an unsigned integer constant by applying a function. ConstantInt *mapUInt(Module *M, ConstantInt *I, std::function F); /// Map a signed integer constant by applying a function. ConstantInt *mapSInt(Module *M, ConstantInt *I, std::function F); /// Get postfix for given decoration. /// The returned postfix does not include "_" at the beginning. std::string getPostfix(Decoration Dec, unsigned Value = 0); /// Get postfix _R{ReturnType} for return type /// The returned postfix does not includ "_" at the beginning std::string getPostfixForReturnType(CallInst *CI, bool IsSigned = false); std::string getPostfixForReturnType(const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); Constant *getScalarOrVectorConstantInt(Type *T, uint64_t V, bool IsSigned = false); /// Get a constant int or a constant int array. /// \param T is the type of the constant. It should be an integer type or // an integer pointer type. /// \param Len is the length of the array. /// \param V is the value to fill the array. Value *getScalarOrArrayConstantInt(Instruction *P, Type *T, unsigned Len, uint64_t V, bool IsSigned = false); /// Get the array from GEP. /// \param V is a GEP whose pointer operand is a pointer to an array of size /// \param Size. Value *getScalarOrArray(Value *V, unsigned Size, Instruction *Pos); void dumpUsers(Value *V, StringRef Prompt = ""); /// Get SPIR-V type name as spirv.BaseTyName.Postfixes. std::string getSPIRVTypeName(StringRef BaseTyName, StringRef Postfixes = ""); /// Checks if given type name is either ConstantSampler or ConsantPipeStorage. bool isSPIRVConstantName(StringRef TyName); /// Get SPIR-V type by changing the type name from spirv.OldName.Postfixes /// to spirv.NewName.Postfixes. Type *getSPIRVTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName); /// Get SPIR-V type by changing the type name from spirv.OldName.Postfixes /// to spirv.NewName.Postfixes. Type *getSPIRVStructTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName); /// Get the postfixes of SPIR-V image type name as in spirv.Image.postfixes. std::string getSPIRVImageTypePostfixes(StringRef SampledType, SPIRVTypeImageDescriptor Desc, SPIRVAccessQualifierKind Acc); /// Get the sampled type name used in postfix of image type in SPIR-V /// friendly LLVM IR. std::string getSPIRVImageSampledTypeName(SPIRVType *Ty); /// Translates OpenCL image type names to SPIR-V. /// E.g. %opencl.image1d_rw_t -> %spirv.Image._void_0_0_0_0_0_0_2 Type *adaptSPIRVImageType(Module *M, Type *PointeeType); /// Get LLVM type for sampled type of SPIR-V image type by postfix. Type *getLLVMTypeForSPIRVImageSampledTypePostfix(StringRef Postfix, LLVMContext &Ctx); /// Return the unqualified and unsuffixed base name of an image type. /// E.g. opencl.image2d_ro_t.3 -> image2d_t std::string getImageBaseTypeName(StringRef Name); /// Map OpenCL opaque type name to SPIR-V type name. std::string mapOCLTypeNameToSPIRV(StringRef Name, StringRef Acc = ""); /// Return the index of image operands given an image op. size_t getImageOperandsIndex(Op OpCode); /// Check if access qualifier is encoded in the type name. bool hasAccessQualifiedName(StringRef TyName); /// Get access qualifier from the type name. SPIRVAccessQualifierKind getAccessQualifier(StringRef TyName); /// Get access qualifier from the type name. StringRef getAccessQualifierPostfix(SPIRVAccessQualifierKind Access); /// Get access qualifier from the type name. StringRef getAccessQualifierFullName(StringRef TyName); bool eraseUselessFunctions(Module *M); /// Erase a function if it is declaration, has internal linkage and has no use. bool eraseIfNoUse(Function *F); void eraseIfNoUse(Value *V); // Check if a mangled type name is unsigned bool isMangledTypeUnsigned(char Mangled); // Check if a mangled type name is signed bool isMangledTypeSigned(char Mangled); // Check if a mangled type name is floating point (except half) bool isMangledTypeFP(char Mangled); // Check if a mangled type name is half bool isMangledTypeHalf(std::string Mangled); // Check if \param I is valid vector size: 2, 3, 4, 8, 16. bool isValidVectorSize(unsigned I); enum class ParamType { FLOAT = 0, SIGNED = 1, UNSIGNED = 2, UNKNOWN = 3 }; ParamType lastFuncParamType(StringRef MangledName); // Check if the last function parameter is signed bool isLastFuncParamSigned(StringRef MangledName); // Check if a mangled function name contains unsigned atomic type bool containsUnsignedAtomicType(StringRef Name); /// Mangle builtin function name. /// \return \param UniqName if \param BtnInfo is null pointer, otherwise /// return IA64 mangled name. std::string mangleBuiltin(StringRef UniqName, ArrayRef ArgTypes, BuiltinFuncMangleInfo *BtnInfo); /// Extract the pointee types of arguments from a mangled function name. If the /// corresponding type is not a pointer to a struct type, its value will be a /// nullptr instead. void getParameterTypes(Function *F, SmallVectorImpl &ArgTys); inline void getParameterTypes(CallInst *CI, SmallVectorImpl &ArgTys) { return getParameterTypes(CI->getCalledFunction(), ArgTys); } /// Mangle a function from OpenCL extended instruction set in SPIR-V friendly IR /// manner std::string getSPIRVFriendlyIRFunctionName(OCLExtOpKind ExtOpId, ArrayRef ArgTys, ArrayRef PETs, Type *RetTy = nullptr); /// Mangle a function in SPIR-V friendly IR manner /// \param UniqName full unmangled name of the SPIR-V built-in function that /// contains possible postfixes that depend not on opcode but on decorations or /// return type, for example __spirv_UConvert_Rint_sat. /// \param OC opcode of corresponding built-in instruction. Used to gather info /// for unsigned/constant arguments. /// \param Types of arguments of SPIR-V built-in function /// \param Ops Operands of SPIRVInstruction /// \return IA64 mangled name. std::string getSPIRVFriendlyIRFunctionName(const std::string &UniqName, spv::Op OC, ArrayRef ArgTys, ArrayRef PETs, ArrayRef Ops); /// Cast a function to a void(void) funtion pointer. Constant *castToVoidFuncPtr(Function *F); /// Get i8* with the same address space. PointerType *getInt8PtrTy(PointerType *T); /// Cast a value to a i8* by inserting a cast instruction. Value *castToInt8Ptr(Value *V, Instruction *Pos); template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x) add(#x, OpType##x); _SPIRV_OP(DeviceEvent) _SPIRV_OP(Event) _SPIRV_OP(Image) _SPIRV_OP(Pipe) _SPIRV_OP(Queue) _SPIRV_OP(ReserveId) _SPIRV_OP(Sampler) _SPIRV_OP(SampledImage) // SPV_INTEL_device_side_avc_motion_estimation types _SPIRV_OP(AvcMcePayloadINTEL) _SPIRV_OP(AvcImePayloadINTEL) _SPIRV_OP(AvcRefPayloadINTEL) _SPIRV_OP(AvcSicPayloadINTEL) _SPIRV_OP(AvcMceResultINTEL) _SPIRV_OP(AvcImeResultINTEL) _SPIRV_OP(AvcImeResultSingleReferenceStreamoutINTEL) _SPIRV_OP(AvcImeResultDualReferenceStreamoutINTEL) _SPIRV_OP(AvcImeSingleReferenceStreaminINTEL) _SPIRV_OP(AvcImeDualReferenceStreaminINTEL) _SPIRV_OP(AvcRefResultINTEL) _SPIRV_OP(AvcSicResultINTEL) _SPIRV_OP(CooperativeMatrixKHR) #undef _SPIRV_OP } // Check if the module contains llvm.loop.* metadata bool hasLoopMetadata(const Module *M); // Check if CI is a call to instruction from OpenCL Extended Instruction Set. // If so, return it's extended opcode in ExtOp. bool isSPIRVOCLExtInst(const CallInst *CI, OCLExtOpKind *ExtOp); // check LLVM Intrinsics type(s) for validity bool checkTypeForSPIRVExtendedInstLowering(IntrinsicInst *II, SPIRVModule *BM); /// Decode SPIR-V type name in the format spirv.{TypeName}._{Postfixes} /// where Postfixes are strings separated by underscores. /// \return TypeName. /// \param Strs contains the integers decoded from postfixes. std::string decodeSPIRVTypeName(StringRef Name, SmallVectorImpl &Strs); // Copy attributes from function to call site. CallInst *setAttrByCalledFunc(CallInst *Call); bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind); // Transform builtin variable from GlobalVariable to builtin call. // e.g. // - GlobalInvolcationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x) // - WorkDim -> _Z22__spirv_BuiltInWorkDimv() bool lowerBuiltinVariableToCall(GlobalVariable *GV, SPIRVBuiltinVariableKind Kind); // Transform all builtin variables into calls bool lowerBuiltinVariablesToCalls(Module *M); /// \brief Post-process OpenCL or SPIRV builtin function returning struct type. /// /// Some builtin functions are translated to SPIR-V instructions with /// struct type result, e.g. NDRange creation functions. Such functions /// need to be post-processed to return the struct through sret argument. bool postProcessBuiltinReturningStruct(Function *F); /// \brief Post-process OpenCL or SPIRV builtin function having array argument. /// /// These functions are translated to functions with array type argument /// first, then post-processed to have pointer arguments. bool postProcessBuiltinWithArrayArguments(Function *F, StringRef DemangledName); bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp = false); bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp = false); template MetadataAsValue *map2MDString(LLVMContext &C, SPIRVValue *V); /// Returns the smallest integral power of two no smaller than Value if Value is /// nonzero. Returns 1 otherwise. /// /// Ex. bitCeil(5) == 8. /// /// The return value is undefined if the input is larger than the largest power /// of two representable in SPIRVWord. [[nodiscard]] SPIRVWord bitCeil(SPIRVWord Value); } // namespace SPIRV #endif // SPIRV_SPIRVINTERNAL_H