///////////////////////////////////////////////////////////////////////////////

//                                                                           //

// HLOperationLower.cpp                                                      //

// Copyright (C) Microsoft Corporation. All rights reserved.                 //

// This file is distributed under the University of Illinois Open Source     //

// License. See LICENSE.TXT for details.                                     //

//                                                                           //

// Lower functions to lower HL operations to DXIL operations.                //

//                                                                           //

///////////////////////////////////////////////////////////////////////////////

#include "dxc/DXIL/DxilConstants.h"

#define _USE_MATH_DEFINES

#include <array>

#include <cmath>

#include <functional>

#include <unordered_set>

#include "dxc/DXIL/DxilConstants.h"

#include "dxc/DXIL/DxilInstructions.h"

#include "dxc/DXIL/DxilModule.h"

#include "dxc/DXIL/DxilOperations.h"

#include "dxc/DXIL/DxilResourceProperties.h"

#include "dxc/DXIL/DxilUtil.h"

#include "dxc/HLSL/DxilPoisonValues.h"

#include "dxc/HLSL/HLLowerUDT.h"

#include "dxc/HLSL/HLMatrixLowerHelper.h"

#include "dxc/HLSL/HLMatrixType.h"

#include "dxc/HLSL/HLModule.h"

#include "dxc/HLSL/HLOperationLower.h"

#include "dxc/HLSL/HLOperationLowerExtension.h"

#include "dxc/HLSL/HLOperations.h"

#include "dxc/HlslIntrinsicOp.h"

#include "llvm/ADT/APSInt.h"

#include "llvm/IR/GetElementPtrTypeIterator.h"

#include "llvm/IR/IRBuilder.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/Module.h"

using namespace llvm;

using namespace hlsl;

struct HLOperationLowerHelper {

  HLModule &M;

  OP &hlslOP;

  Type *voidTy;

  Type *f32Ty;

  Type *i32Ty;

  Type *i16Ty;

  llvm::Type *i1Ty;

  Type *i8Ty;

  DxilTypeSystem &dxilTypeSys;

  DxilFunctionProps *functionProps;

  DataLayout dataLayout;

  SmallDenseMap<Type *, Type *, 4> loweredTypes;

  HLOperationLowerHelper(HLModule &HLM);

};

HLOperationLowerHelper::HLOperationLowerHelper(HLModule &HLM)

    : M(HLM), hlslOP(*HLM.GetOP()), dxilTypeSys(HLM.GetTypeSystem()),

      dataLayout(DataLayout(HLM.GetHLOptions().bUseMinPrecision

                                ? 
hlsl::DXIL::kLegacyLayoutString38.6k

                                : 
hlsl::DXIL::kNewLayoutString2.24k
)) {

  llvm::LLVMContext &Ctx = HLM.GetCtx();

  voidTy = Type::getVoidTy(Ctx);

  f32Ty = Type::getFloatTy(Ctx);

  i32Ty = Type::getInt32Ty(Ctx);

  i16Ty = Type::getInt16Ty(Ctx);

  i1Ty = Type::getInt1Ty(Ctx);

  i8Ty = Type::getInt8Ty(Ctx);

  Function *EntryFunc = HLM.GetEntryFunction();

  functionProps = nullptr;

  if (HLM.HasDxilFunctionProps(EntryFunc))

    functionProps = &HLM.GetDxilFunctionProps(EntryFunc);

}

struct HLObjectOperationLowerHelper {

private:

  // For object intrinsics.

  HLModule &HLM;

  struct ResAttribute {

    DXIL::ResourceClass RC;

    DXIL::ResourceKind RK;

    Type *ResourceType;

  };

  std::unordered_map<Value *, ResAttribute> HandleMetaMap;

  std::unordered_set<Instruction *> &UpdateCounterSet;

  // Map from pointer of cbuffer to pointer of resource.

  // For cbuffer like this:

  //   cbuffer A {

  //     Texture2D T;

  //   };

  // A global resource Texture2D T2 will be created for Texture2D T.

  // CBPtrToResourceMap[T] will return T2.

  std::unordered_map<Value *, Value *> CBPtrToResourceMap;

public:

  HLObjectOperationLowerHelper(HLModule &HLM,

                               std::unordered_set<Instruction *> &UpdateCounter)

      : HLM(HLM), UpdateCounterSet(UpdateCounter) {}

  DXIL::ResourceClass GetRC(Value *Handle) {

    ResAttribute &Res = FindCreateHandleResourceBase(Handle);

    return Res.RC;

  }

  DXIL::ResourceKind GetRK(Value *Handle) {

    ResAttribute &Res = FindCreateHandleResourceBase(Handle);

    return Res.RK;

  }

  Type *GetResourceType(Value *Handle) {

    ResAttribute &Res = FindCreateHandleResourceBase(Handle);

    return Res.ResourceType;

  }

  void MarkHasCounter(Value *handle, Type *i8Ty) {

    CallInst *CIHandle = cast<CallInst>(handle);

    DXASSERT(hlsl::GetHLOpcodeGroup(CIHandle->getCalledFunction()) ==

                 HLOpcodeGroup::HLAnnotateHandle,

             "else invalid handle");

    // Mark has counter for the input handle.

    Value *counterHandle =

        CIHandle->getArgOperand(HLOperandIndex::kHandleOpIdx);

    // Change kind into StructurBufferWithCounter.

    Constant *Props = cast<Constant>(CIHandle->getArgOperand(

        HLOperandIndex::kAnnotateHandleResourcePropertiesOpIdx));

    DxilResourceProperties RP = resource_helper::loadPropsFromConstant(*Props);

    RP.Basic.SamplerCmpOrHasCounter = true;

    CIHandle->setArgOperand(

        HLOperandIndex::kAnnotateHandleResourcePropertiesOpIdx,

        resource_helper::getAsConstant(RP,

                                       HLM.GetOP()->GetResourcePropertiesType(),

                                       *HLM.GetShaderModel()));

    DXIL::ResourceClass RC = GetRC(handle);

    DXASSERT_LOCALVAR(RC, RC == DXIL::ResourceClass::UAV,

                      "must UAV for counter");

    std::unordered_set<Value *> resSet;

    MarkHasCounterOnCreateHandle(counterHandle, resSet);

  }

  DxilResourceBase *FindCBufferResourceFromHandle(Value *handle) {

    if (CallInst *CI = dyn_cast<CallInst>(handle)) {

      hlsl::HLOpcodeGroup group =

          hlsl::GetHLOpcodeGroupByName(CI->getCalledFunction());

      if (group == HLOpcodeGroup::HLAnnotateHandle) {

        handle = CI->getArgOperand(HLOperandIndex::kHandleOpIdx);

      }

    }

    Constant *symbol = nullptr;

    if (CallInst *CI = dyn_cast<CallInst>(handle)) {

      hlsl::HLOpcodeGroup group =

          hlsl::GetHLOpcodeGroupByName(CI->getCalledFunction());

      if (group == HLOpcodeGroup::HLCreateHandle) {

        symbol = dyn_cast<Constant>(

            CI->getArgOperand(HLOperandIndex::kCreateHandleResourceOpIdx));

      }

    }

    if (!symbol)

      return nullptr;

    for (const std::unique_ptr<DxilCBuffer> &res : HLM.GetCBuffers()) {

      if (res->GetGlobalSymbol() == symbol)

        return res.get();

    }

    return nullptr;

  }

  Value *GetOrCreateResourceForCbPtr(GetElementPtrInst *CbPtr,

                                     GlobalVariable *CbGV,

                                     DxilResourceProperties &RP) {

    // Change array idx to 0 to make sure all array ptr share same key.

    Value *Key = UniformCbPtr(CbPtr, CbGV);

    if (CBPtrToResourceMap.count(Key))

      return CBPtrToResourceMap[Key];

    Value *Resource = CreateResourceForCbPtr(CbPtr, CbGV, RP);

    CBPtrToResourceMap[Key] = Resource;

    return Resource;

  }

  Value *LowerCbResourcePtr(GetElementPtrInst *CbPtr, Value *ResPtr) {

    // Simple case.

    if (ResPtr->getType() == CbPtr->getType())

      return ResPtr;

    // Array case.

    DXASSERT_NOMSG(ResPtr->getType()->getPointerElementType()->isArrayTy());

    IRBuilder<> Builder(CbPtr);

    gep_type_iterator GEPIt = gep_type_begin(CbPtr), E = gep_type_end(CbPtr);

    Value *arrayIdx = GEPIt.getOperand();

    // Only calc array idx and size.

    // Ignore struct type part.

    for (; GEPIt != E; ++GEPIt) {

      if (GEPIt->isArrayTy()) {

        arrayIdx = Builder.CreateMul(

            arrayIdx, Builder.getInt32(GEPIt->getArrayNumElements()));

        arrayIdx = Builder.CreateAdd(arrayIdx, GEPIt.getOperand());

      }

    }

    return Builder.CreateGEP(ResPtr, {Builder.getInt32(0), arrayIdx});

  }

  DxilResourceProperties GetResPropsFromAnnotateHandle(CallInst *Anno) {

    Constant *Props = cast<Constant>(Anno->getArgOperand(

        HLOperandIndex::kAnnotateHandleResourcePropertiesOpIdx));

    DxilResourceProperties RP = resource_helper::loadPropsFromConstant(*Props);

    return RP;

  }

private:

  ResAttribute &FindCreateHandleResourceBase(Value *Handle) {

    if (HandleMetaMap.count(Handle))

      return HandleMetaMap[Handle];

    // Add invalid first to avoid dead loop.

    HandleMetaMap[Handle] = {

        DXIL::ResourceClass::Invalid, DXIL::ResourceKind::Invalid,

        StructType::get(Type::getVoidTy(HLM.GetCtx()), nullptr)};

    if (CallInst *CI = dyn_cast<CallInst>(Handle)) {

      hlsl::HLOpcodeGroup group =

          hlsl::GetHLOpcodeGroupByName(CI->getCalledFunction());

      if (group == HLOpcodeGroup::HLAnnotateHandle) {

        Constant *Props = cast<Constant>(CI->getArgOperand(

            HLOperandIndex::kAnnotateHandleResourcePropertiesOpIdx));

        DxilResourceProperties RP =

            resource_helper::loadPropsFromConstant(*Props);

        Type *ResTy =

            CI->getArgOperand(HLOperandIndex::kAnnotateHandleResourceTypeOpIdx)

                ->getType();

        ResAttribute Attrib = {RP.getResourceClass(), RP.getResourceKind(),

                               ResTy};

        HandleMetaMap[Handle] = Attrib;

        return HandleMetaMap[Handle];

      }

    }

    dxilutil::EmitErrorOnContext(Handle->getContext(),

                                 "cannot map resource to handle.");

    return HandleMetaMap[Handle];

  }

  CallInst *FindCreateHandle(Value *handle,

                             std::unordered_set<Value *> &resSet) {

    // Already checked.

    if (resSet.count(handle))

      return nullptr;

    resSet.insert(handle);

    if (CallInst *CI = dyn_cast<CallInst>(handle))

      return CI;

    if (SelectInst *Sel = dyn_cast<SelectInst>(handle)) {

      if (CallInst *CI = FindCreateHandle(Sel->getTrueValue(), resSet))

        return CI;

      if (CallInst *CI = FindCreateHandle(Sel->getFalseValue(), resSet))

        return CI;

      return nullptr;

    }

    if (PHINode *Phi = dyn_cast<PHINode>(handle)) {

      for (unsigned i = 0; i < Phi->getNumOperands(); i++) {

        if (CallInst *CI = FindCreateHandle(Phi->getOperand(i), resSet))

          return CI;

      }

      return nullptr;

    }

    return nullptr;

  }

  void MarkHasCounterOnCreateHandle(Value *handle,

                                    std::unordered_set<Value *> &resSet) {

    // Already checked.

    if (resSet.count(handle))

      return;

    resSet.insert(handle);

    if (CallInst *CI = dyn_cast<CallInst>(handle)) {

      Value *Res =

          CI->getArgOperand(HLOperandIndex::kCreateHandleResourceOpIdx);

      LoadInst *LdRes = dyn_cast<LoadInst>(Res);

      if (LdRes) {

        UpdateCounterSet.insert(LdRes);

        return;

      }

      if (CallInst *CallRes = dyn_cast<CallInst>(Res)) {

        hlsl::HLOpcodeGroup group =

            hlsl::GetHLOpcodeGroup(CallRes->getCalledFunction());

        if (group == HLOpcodeGroup::HLCast) {

          HLCastOpcode opcode =

              static_cast<HLCastOpcode>(hlsl::GetHLOpcode(CallRes));

          if (opcode == HLCastOpcode::HandleToResCast) {

            if (Instruction *Hdl = dyn_cast<Instruction>(

                    CallRes->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx)))

              UpdateCounterSet.insert(Hdl);

            return;

          }

        }

      }

      dxilutil::EmitErrorOnInstruction(CI, "cannot map resource to handle.");

      return;

    }

    if (SelectInst *Sel = dyn_cast<SelectInst>(handle)) {

      MarkHasCounterOnCreateHandle(Sel->getTrueValue(), resSet);

      MarkHasCounterOnCreateHandle(Sel->getFalseValue(), resSet);

    }

    if (PHINode *Phi = dyn_cast<PHINode>(handle)) {

      for (unsigned i = 0; i < Phi->getNumOperands(); i++) {

        MarkHasCounterOnCreateHandle(Phi->getOperand(i), resSet);

      }

    }

  }

  Value *UniformCbPtr(GetElementPtrInst *CbPtr, GlobalVariable *CbGV) {

    gep_type_iterator GEPIt = gep_type_begin(CbPtr), E = gep_type_end(CbPtr);

    std::vector<Value *> idxList(CbPtr->idx_begin(), CbPtr->idx_end());

    unsigned i = 0;

    IRBuilder<> Builder(HLM.GetCtx());

    Value *zero = Builder.getInt32(0);

    for (; GEPIt != E; 
++GEPIt, ++i982
) {

      ConstantInt *ImmIdx = dyn_cast<ConstantInt>(GEPIt.getOperand());

      if (!ImmIdx) {

        // Remove dynamic indexing to avoid crash.

        idxList[i] = zero;

      }

    }

    Value *Key = Builder.CreateInBoundsGEP(CbGV, idxList);

    return Key;

  }

  Value *CreateResourceForCbPtr(GetElementPtrInst *CbPtr, GlobalVariable *CbGV,

                                DxilResourceProperties &RP) {

    Type *CbTy = CbPtr->getPointerOperandType();

    DXASSERT_LOCALVAR(CbTy, CbTy == CbGV->getType(),

                      "else arg not point to var");

    gep_type_iterator GEPIt = gep_type_begin(CbPtr), E = gep_type_end(CbPtr);

    unsigned i = 0;

    IRBuilder<> Builder(HLM.GetCtx());

    unsigned arraySize = 1;

    DxilTypeSystem &typeSys = HLM.GetTypeSystem();

    std::string Name;

    for (; GEPIt != E; 
++GEPIt, ++i902
) {

      if (GEPIt->isArrayTy()) {

        arraySize *= GEPIt->getArrayNumElements();

        if (!Name.empty())

          Name += ".";

        if (ConstantInt *ImmIdx = dyn_cast<ConstantInt>(GEPIt.getOperand())) {

          unsigned idx = ImmIdx->getLimitedValue();

          Name += std::to_string(idx);

        }

      } else if (GEPIt->isStructTy()) {

        DxilStructAnnotation *typeAnnot =

            typeSys.GetStructAnnotation(cast<StructType>(*GEPIt));

        DXASSERT_NOMSG(typeAnnot);

        unsigned idx = cast<ConstantInt>(GEPIt.getOperand())->getLimitedValue();

        DXASSERT_NOMSG(typeAnnot->GetNumFields() > idx);

        DxilFieldAnnotation &fieldAnnot = typeAnnot->GetFieldAnnotation(idx);

        if (!Name.empty())

          Name += ".";

        Name += fieldAnnot.GetFieldName();

      }

    }

    Type *Ty = CbPtr->getResultElementType();

    // Not support resource array in cbuffer.

    unsigned ResBinding =

        HLM.GetBindingForResourceInCB(CbPtr, CbGV, RP.getResourceClass());

    return CreateResourceGV(Ty, Name, RP, ResBinding);

  }

  Value *CreateResourceGV(Type *Ty, StringRef Name, DxilResourceProperties &RP,

                          unsigned ResBinding) {

    Module &M = *HLM.GetModule();

    Constant *GV = M.getOrInsertGlobal(Name, Ty);

    // Create resource and set GV as globalSym.

    DxilResourceBase *Res = HLM.AddResourceWithGlobalVariableAndProps(GV, RP);

    DXASSERT(Res, "fail to create resource for global variable in cbuffer");

    Res->SetLowerBound(ResBinding);

    return GV;

  }

};

// Helper for lowering resource extension methods.

struct HLObjectExtensionLowerHelper : public hlsl::HLResourceLookup {

  explicit HLObjectExtensionLowerHelper(HLObjectOperationLowerHelper &ObjHelper)

      : m_ObjHelper(ObjHelper) {}

  virtual bool GetResourceKindName(Value *HLHandle, const char **ppName) {

    DXIL::ResourceKind K = m_ObjHelper.GetRK(HLHandle);

    bool Success = K != DXIL::ResourceKind::Invalid;

    if (Success) {

      *ppName = hlsl::GetResourceKindName(K);

    }

    return Success;

  }

private:

  HLObjectOperationLowerHelper &m_ObjHelper;

};

using IntrinsicLowerFuncTy = Value *(CallInst *CI, IntrinsicOp IOP,

                                     DXIL::OpCode opcode,

                                     HLOperationLowerHelper &helper,

                                     HLObjectOperationLowerHelper *pObjHelper,

                                     bool &Translated);

struct IntrinsicLower {

  // Intrinsic opcode.

  IntrinsicOp IntriOpcode;

  // Lower function.

  IntrinsicLowerFuncTy &LowerFunc;

  // DXIL opcode if can direct map.

  DXIL::OpCode DxilOpcode;

};

// IOP intrinsics.

namespace {

// Creates the necessary scalar calls to for a "trivial" operation where only

// call instructions to a single function type are needed.

// The overload type `Ty` determines what scalarization might be required.

// Elements of any vectors in `refArgs` are extracted  into scalars for each

// call generated while the same scalar values are used unaltered in each call.

// Utility objects `HlslOp` and `Builder` are used to generate calls to the

// given `DxilFunc` for each set of scalar arguments.

// The results are reconstructed into the given `RetTy` as needed.

Value *TrivialDxilOperation(Function *dxilFunc, OP::OpCode opcode,

                            ArrayRef<Value *> refArgs, Type *Ty, Type *RetTy,

                            OP *hlslOP, IRBuilder<> &Builder) {

  unsigned argNum = refArgs.size();

  std::vector<Value *> args = refArgs;

  if (Ty->isVectorTy()) {

    Value *retVal = llvm::UndefValue::get(RetTy);

    unsigned vecSize = Ty->getVectorNumElements();

    for (unsigned i = 0; i < vecSize; 
i++26.6k
) {

      // Update vector args, skip known opcode arg.

      for (unsigned argIdx = HLOperandIndex::kUnaryOpSrc0Idx; argIdx < argNum;

           argIdx++) {

        if (refArgs[argIdx]->getType()->isVectorTy()) {

          Value *arg = refArgs[argIdx];

          args[argIdx] = Builder.CreateExtractElement(arg, i);

        }

      }

      Value *EltOP =

          Builder.CreateCall(dxilFunc, args, hlslOP->GetOpCodeName(opcode));

      retVal = Builder.CreateInsertElement(retVal, EltOP, i);

    }

    return retVal;

  }

  // Cannot add name to void.

  if (RetTy->isVoidTy())

    return Builder.CreateCall(dxilFunc, args);

  return Builder.CreateCall(dxilFunc, args, hlslOP->GetOpCodeName(opcode));

}

// Creates a native vector call to for a "trivial" operation where only a single

// call instruction is needed. The overload and return types are the same vector

// type `Ty`.

// Utility objects `HlslOp` and `Builder` are used to create a call to the given

// `DxilFunc` with `RefArgs` arguments.

Value *TrivialDxilVectorOperation(Function *Func, OP::OpCode Opcode,

                                  ArrayRef<Value *> Args, Type *Ty, OP *OP,

                                  IRBuilder<> &Builder) {

  if (!Ty->isVoidTy())

    return Builder.CreateCall(Func, Args, OP->GetOpCodeName(Opcode));

  return Builder.CreateCall(Func, Args); // Cannot add name to void.

}

// Generates a DXIL operation with the overloaded type based on `Ty` and return

// type `RetTy`. When Ty is a vector, it will either generate per-element calls

// for each vector element and reconstruct the vector type from those results or

// operate on and return native vectors depending on vector size and the

// legality of the vector overload.

Value *TrivialDxilOperation(OP::OpCode opcode, ArrayRef<Value *> refArgs,

                            Type *Ty, Type *RetTy, OP *hlslOP,

                            IRBuilder<> &Builder) {

  // If supported and the overload type is a vector with more than 1 element,

  // create a native vector operation.

  if (Ty->isVectorTy() && 
Ty->getVectorNumElements() > 19.33k
 &&

      
hlslOP->GetModule()->GetHLModule().GetShaderModel()->IsSM69Plus()8.51k
 &&

      
OP::IsOverloadLegal(opcode, Ty)1.08k
) {

    Function *dxilFunc = hlslOP->GetOpFunc(opcode, Ty);

    return TrivialDxilVectorOperation(dxilFunc, opcode, refArgs, Ty, hlslOP,

                                      Builder);

  }

  // Set overload type to the scalar type of `Ty` and generate call(s).

  Type *EltTy = Ty->getScalarType();

  Function *dxilFunc = hlslOP->GetOpFunc(opcode, EltTy);

  return TrivialDxilOperation(dxilFunc, opcode, refArgs, Ty, RetTy, hlslOP,

                              Builder);

}

Value *TrivialDxilOperation(OP::OpCode opcode, ArrayRef<Value *> refArgs,

                            Type *Ty, Instruction *Inst, OP *hlslOP) {

  DXASSERT(refArgs.size() > 0, "else opcode isn't in signature");

  DXASSERT(refArgs[0] == nullptr,

           "else caller has already filled the value in");

  IRBuilder<> B(Inst);

  Constant *opArg = hlslOP->GetU32Const((unsigned)opcode);

  const_cast<llvm::Value **>(refArgs.data())[0] =

      opArg; // actually stack memory from caller

  return TrivialDxilOperation(opcode, refArgs, Ty, Inst->getType(), hlslOP, B);

}

// Translate call that converts to a dxil unary operation with a different

// return type from the overload by passing the argument, explicit return type,

// and helper objects to the scalarizing unary dxil operation creation.

Value *TrivialUnaryOperationRet(CallInst *CI, IntrinsicOp IOP,

                                OP::OpCode OpCode,

                                HLOperationLowerHelper &Helper,

                                HLObjectOperationLowerHelper *,

                                bool &Translated) {

  Value *Src = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  Type *Ty = Src->getType();

  IRBuilder<> Builder(CI);

  hlsl::OP *OP = &Helper.hlslOP;

  Type *RetTy = CI->getType();

  Constant *OpArg = OP->GetU32Const((unsigned)OpCode);

  Value *Args[] = {OpArg, Src};

  return TrivialDxilOperation(OpCode, Args, Ty, RetTy, OP, Builder);

}

Value *TrivialDxilUnaryOperation(OP::OpCode OpCode, Value *Src, hlsl::OP *Op,

                                 IRBuilder<> &Builder) {

  Type *Ty = Src->getType();

  Constant *OpArg = Op->GetU32Const((unsigned)OpCode);

  Value *Args[] = {OpArg, Src};

  return TrivialDxilOperation(OpCode, Args, Ty, Ty, Op, Builder);

}

Value *TrivialDxilBinaryOperation(OP::OpCode opcode, Value *src0, Value *src1,

                                  hlsl::OP *hlslOP, IRBuilder<> &Builder) {

  Type *Ty = src0->getType();

  Constant *opArg = hlslOP->GetU32Const((unsigned)opcode);

  Value *args[] = {opArg, src0, src1};

  return TrivialDxilOperation(opcode, args, Ty, Ty, hlslOP, Builder);

}

Value *TrivialDxilTrinaryOperation(OP::OpCode opcode, Value *src0, Value *src1,

                                   Value *src2, hlsl::OP *hlslOP,

                                   IRBuilder<> &Builder) {

  Type *Ty = src0->getType();

  Constant *opArg = hlslOP->GetU32Const((unsigned)opcode);

  Value *args[] = {opArg, src0, src1, src2};

  return TrivialDxilOperation(opcode, args, Ty, Ty, hlslOP, Builder);

}

// Translate call that trivially converts to a dxil unary operation by passing

// argument, return type, and helper objects to either scalarizing or native

// vector dxil operation creation depending on version and vector size.

Value *TrivialUnaryOperation(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                             HLOperationLowerHelper &helper,

                             HLObjectOperationLowerHelper *pObjHelper,

                             bool &Translated) {

  Value *src0 = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  IRBuilder<> Builder(CI);

  hlsl::OP *hlslOP = &helper.hlslOP;

  return TrivialDxilUnaryOperation(opcode, src0, hlslOP, Builder);

}

// Translate call that trivially converts to a dxil binary operation by passing

// arguments, return type, and helper objects to either scalarizing or native

// vector dxil operation creation depending on version and vector size.

Value *TrivialBinaryOperation(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                              HLOperationLowerHelper &helper,

                              HLObjectOperationLowerHelper *pObjHelper,

                              bool &Translated) {

  hlsl::OP *hlslOP = &helper.hlslOP;

  Value *src0 = CI->getArgOperand(HLOperandIndex::kBinaryOpSrc0Idx);

  Value *src1 = CI->getArgOperand(HLOperandIndex::kBinaryOpSrc1Idx);

  IRBuilder<> Builder(CI);

  Value *binOp =

      TrivialDxilBinaryOperation(opcode, src0, src1, hlslOP, Builder);

  return binOp;

}

// Translate call that trivially converts to a dxil trinary (aka tertiary)

// operation by passing arguments, return type, and helper objects to either

// scalarizing or native vector dxil operation creation depending on version

// and vector size.

Value *TrivialTrinaryOperation(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                               HLOperationLowerHelper &helper,

                               HLObjectOperationLowerHelper *pObjHelper,

                               bool &Translated) {

  hlsl::OP *hlslOP = &helper.hlslOP;

  Value *src0 = CI->getArgOperand(HLOperandIndex::kTrinaryOpSrc0Idx);

  Value *src1 = CI->getArgOperand(HLOperandIndex::kTrinaryOpSrc1Idx);

  Value *src2 = CI->getArgOperand(HLOperandIndex::kTrinaryOpSrc2Idx);

  IRBuilder<> Builder(CI);

  Value *triOp =

      TrivialDxilTrinaryOperation(opcode, src0, src1, src2, hlslOP, Builder);

  return triOp;

}

Value *TrivialIsSpecialFloat(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                             HLOperationLowerHelper &helper,

                             HLObjectOperationLowerHelper *pObjHelper,

                             bool &Translated) {

  hlsl::OP *hlslOP = &helper.hlslOP;

  Value *src = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  IRBuilder<> Builder(CI);

  Type *Ty = src->getType();

  Type *RetTy = Type::getInt1Ty(CI->getContext());

  if (Ty->isVectorTy())

    RetTy = VectorType::get(RetTy, Ty->getVectorNumElements());

  Constant *opArg = hlslOP->GetU32Const((unsigned)opcode);

  Value *args[] = {opArg, src};

  return TrivialDxilOperation(opcode, args, Ty, RetTy, hlslOP, Builder);

}

bool IsResourceGEP(GetElementPtrInst *I) {

  Type *Ty = I->getType()->getPointerElementType();

  Ty = dxilutil::GetArrayEltTy(Ty);

  // Only mark on GEP which point to resource.

  return dxilutil::IsHLSLResourceType(Ty);

}

Value *TranslateNonUniformResourceIndex(

    CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

    HLOperationLowerHelper &helper, HLObjectOperationLowerHelper *pObjHelper,

    bool &Translated) {

  Value *V = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  Type *hdlTy = helper.hlslOP.GetHandleType();

  for (User *U : CI->users()) {

    if (GetElementPtrInst *I = dyn_cast<GetElementPtrInst>(U)) {

      // Only mark on GEP which point to resource.

      if (IsResourceGEP(I))

        DxilMDHelper::MarkNonUniform(I);

    } else 
if (CastInst *104
castI104
 = dyn_cast<CastInst>(U)) {

      for (User *castU : castI->users()) {

        if (GetElementPtrInst *I = dyn_cast<GetElementPtrInst>(castU)) {

          // Only mark on GEP which point to resource.

          if (IsResourceGEP(I))

            DxilMDHelper::MarkNonUniform(I);

        } else if (CallInst *CI = dyn_cast<CallInst>(castU)) {

          if (CI->getType() == hdlTy)

            DxilMDHelper::MarkNonUniform(CI);

        }

      }

    } else if (CallInst *CI = dyn_cast<CallInst>(U)) {

      if (CI->getType() == hdlTy)

        DxilMDHelper::MarkNonUniform(CI);

    }

  }

  CI->replaceAllUsesWith(V);

  return nullptr;

}

Value *TrivialBarrier(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                      HLOperationLowerHelper &helper,

                      HLObjectOperationLowerHelper *pObjHelper,

                      bool &Translated) {

  hlsl::OP *OP = &helper.hlslOP;

  Function *dxilFunc = OP->GetOpFunc(OP::OpCode::Barrier, CI->getType());

  Constant *opArg = OP->GetU32Const((unsigned)OP::OpCode::Barrier);

  unsigned uglobal = static_cast<unsigned>(DXIL::BarrierMode::UAVFenceGlobal);

  unsigned g = static_cast<unsigned>(DXIL::BarrierMode::TGSMFence);

  unsigned t = static_cast<unsigned>(DXIL::BarrierMode::SyncThreadGroup);

  // unsigned ut =

  // static_cast<unsigned>(DXIL::BarrierMode::UAVFenceThreadGroup);

  unsigned barrierMode = 0;

  switch (IOP) {

  case IntrinsicOp::IOP_AllMemoryBarrier:

    barrierMode = uglobal | g;

    break;

  case IntrinsicOp::IOP_AllMemoryBarrierWithGroupSync:

    barrierMode = uglobal | g | t;

    break;

  case IntrinsicOp::IOP_GroupMemoryBarrier:

    barrierMode = g;

    break;

  case IntrinsicOp::IOP_GroupMemoryBarrierWithGroupSync:

    barrierMode = g | t;

    break;

  case IntrinsicOp::IOP_DeviceMemoryBarrier:

    barrierMode = uglobal;

    break;

  case IntrinsicOp::IOP_DeviceMemoryBarrierWithGroupSync:

    barrierMode = uglobal | t;

    break;

  default:

    DXASSERT(0, "invalid opcode for barrier");

    break;

  }

  Value *src0 = OP->GetU32Const(static_cast<unsigned>(barrierMode));

  Value *args[] = {opArg, src0};

  IRBuilder<> Builder(CI);

  Builder.CreateCall(dxilFunc, args);

  return nullptr;

}

Value *TranslateD3DColorToUByte4(CallInst *CI, IntrinsicOp IOP,

                                 OP::OpCode opcode,

                                 HLOperationLowerHelper &helper,

                                 HLObjectOperationLowerHelper *pObjHelper,

                                 bool &Translated) {

  IRBuilder<> Builder(CI);

  Value *val = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  Type *Ty = val->getType();

  // Use the same scaling factor used by FXC (i.e., 255.001953)

  // Excerpt from stackoverflow discussion:

  // "Built-in rounding, necessary because of truncation. 0.001953 * 256 = 0.5"

  Constant *toByteConst = ConstantFP::get(Ty->getScalarType(), 255.001953);

  if (Ty->isVectorTy()) {

    static constexpr int supportedVecElemCount = 4;

    if (Ty->getVectorNumElements() != supportedVecElemCount) {

      llvm_unreachable(

          "Unsupported input type for intrinsic D3DColorToUByte4.");

      return UndefValue::get(CI->getType());

    }

    toByteConst = ConstantVector::getSplat(supportedVecElemCount, toByteConst);

    // Swizzle the input val -> val.zyxw

    SmallVector<int, 4> mask{2, 1, 0, 3};

    val = Builder.CreateShuffleVector(val, val, mask);

  }

  Value *byte4 = Builder.CreateFMul(toByteConst, val);

  return Builder.CreateCast(Instruction::CastOps::FPToSI, byte4, CI->getType());

}

// Returns true if pow can be implemented using Fxc's mul-only code gen pattern.

// Fxc uses the below rules when choosing mul-only code gen pattern to implement

// pow function. Rule 1: Applicable only to power values in the range

// [INT32_MIN, INT32_MAX] Rule 2: The maximum number of mul ops needed shouldn't

// exceed (2n+1) or (n+1) based on whether the power

//         is a positive or a negative value. Here "n" is the number of scalar

//         elements in power.

// Rule 3: Power must be an exact value.

// +----------+---------------------+------------------+

// | BaseType | IsExponentPositive  | MaxMulOpsAllowed |

// +----------+---------------------+------------------+

// | float4x4 | True                |               33 |

// | float4x4 | False               |               17 |

// | float4x2 | True                |               17 |

// | float4x2 | False               |                9 |

// | float2x4 | True                |               17 |

// | float2x4 | False               |                9 |

// | float4   | True                |                9 |

// | float4   | False               |                5 |

// | float2   | True                |                5 |

// | float2   | False               |                3 |

// | float    | True                |                3 |

// | float    | False               |                2 |

// +----------+---------------------+------------------+

bool CanUseFxcMulOnlyPatternForPow(IRBuilder<> &Builder, Value *x, Value *pow,

                                   int32_t &powI) {

  // Applicable only when power is a literal.

  if (!isa<ConstantDataVector>(pow) && 
!isa<ConstantFP>(pow)262
) {

    return false;

  }

  // Only apply this code gen on splat values.

  if (ConstantDataVector *cdv = dyn_cast<ConstantDataVector>(pow)) {

    if (!hlsl::dxilutil::IsSplat(cdv)) {

      return false;

    }

  }

  // Only apply on aggregates of 16 or fewer elements,

  // representing the max 4x4 matrix size.

  Type *Ty = x->getType();

  if (Ty->isVectorTy() && 
Ty->getVectorNumElements() > 161.18k
)

    return false;

  APFloat powAPF = isa<ConstantDataVector>(pow)

                       ? 
cast<ConstantDataVector>(pow)->getElementAsAPFloat(0)1.18k

                       : // should be a splat value

                       
cast<ConstantFP>(pow)->getValueAPF()188
;

  APSInt powAPS(32, false);

  bool isExact = false;

  // Try converting float value of power to integer and also check if the float

  // value is exact.

  APFloat::opStatus status =

      powAPF.convertToInteger(powAPS, APFloat::rmTowardZero, &isExact);

  if (status == APFloat::opStatus::opOK && 
isExact348
) {

    powI = powAPS.getExtValue();

    uint32_t powU = abs(powI);

    int setBitCount = 0;

    int maxBitSetPos = -1;

    for (int i = 0; i < 32; 
i++10.8k
) {

      if ((powU >> i) & 1) {

        setBitCount++;

        maxBitSetPos = i;

      }

    }

    DXASSERT(maxBitSetPos <= 30, "msb should always be zero.");

    unsigned numElem =

        isa<ConstantDataVector>(pow) ? 
x->getType()->getVectorNumElements()152
 : 
1188
;

    int mulOpThreshold = powI < 0 ? 
numElem + 132
 : 
2 * numElem + 1308
;

    int mulOpNeeded = maxBitSetPos + setBitCount - 1;

    return mulOpNeeded <= mulOpThreshold;

  }

  return false;

}

Value *TranslatePowUsingFxcMulOnlyPattern(IRBuilder<> &Builder, Value *x,

                                          const int32_t y) {

  uint32_t absY = abs(y);

  // If y is zero then always return 1.

  if (absY == 0) {

    return ConstantFP::get(x->getType(), 1);

  }

  int lastSetPos = -1;

  Value *result = nullptr;

  Value *mul = nullptr;

  for (int i = 0; i < 32; 
i++5.63k
) {

    if ((absY >> i) & 1) {

      for (int j = i; j > lastSetPos; 
j--1.00k
) {

        if (!mul) {

          mul = x;

        } else {

          mul = Builder.CreateFMul(mul, mul);

        }

      }

      result = (result == nullptr) ? 
mul176
 : 
Builder.CreateFMul(result, mul)128
;

      lastSetPos = i;

    }

  }

  // Compute reciprocal for negative power values.

  if (y < 0) {

    Value *constOne = ConstantFP::get(x->getType(), 1);

    result = Builder.CreateFDiv(constOne, result);

  }

  return result;

}

Value *TranslatePowImpl(hlsl::OP *hlslOP, IRBuilder<> &Builder, Value *x,

                        Value *y, bool isFXCCompatMode = false) {

  // As applicable implement pow using only mul ops as done by Fxc.

  int32_t p = 0;

  if (CanUseFxcMulOnlyPatternForPow(Builder, x, y, p)) {

    if (isFXCCompatMode)

      return TranslatePowUsingFxcMulOnlyPattern(Builder, x, p);

    // Only take care 2 for it will not affect register pressure.

    if (p == 2)

      return Builder.CreateFMul(x, x);

  }

  // Default to log-mul-exp pattern if previous scenarios don't apply.

  // t = log(x);

  Value *logX =

      TrivialDxilUnaryOperation(DXIL::OpCode::Log, x, hlslOP, Builder);

  // t = y * t;

  Value *mulY = Builder.CreateFMul(logX, y);

  // pow = exp(t);

  return TrivialDxilUnaryOperation(DXIL::OpCode::Exp, mulY, hlslOP, Builder);

}

Value *TranslateAddUint64(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

                          HLOperationLowerHelper &helper,

                          HLObjectOperationLowerHelper *pObjHelper,

                          bool &Translated) {

  hlsl::OP *hlslOP = &helper.hlslOP;

  IRBuilder<> Builder(CI);

  Value *val = CI->getArgOperand(HLOperandIndex::kUnaryOpSrc0Idx);

  Type *Ty = val->getType();

  VectorType *VT = dyn_cast<VectorType>(Ty);

  if (!VT) {

    dxilutil::EmitErrorOnInstruction(

        CI, "AddUint64 can only be applied to uint2 and uint4 operands.");

    return UndefValue::get(Ty);

  }

  unsigned size = VT->getNumElements();

  if (size != 2 && 
size != 424
) {

    dxilutil::EmitErrorOnInstruction(

        CI, "AddUint64 can only be applied to uint2 and uint4 operands.");

    return UndefValue::get(Ty);

  }

  Value *op0 = CI->getArgOperand(HLOperandIndex::kBinaryOpSrc0Idx);

  Value *op1 = CI->getArgOperand(HLOperandIndex::kBinaryOpSrc1Idx);

  Value *RetVal = UndefValue::get(Ty);

  Function *AddC = hlslOP->GetOpFunc(DXIL::OpCode::UAddc, helper.i32Ty);

  Value *opArg = Builder.getInt32(static_cast<unsigned>(DXIL::OpCode::UAddc));

  for (unsigned i = 0; i < size; 
i += 224
) {

    Value *low0 = Builder.CreateExtractElement(op0, i);

    Value *low1 = Builder.CreateExtractElement(op1, i);

    Value *lowWithC = Builder.CreateCall(AddC, {opArg, low0, low1});

    Value *low = Builder.CreateExtractValue(lowWithC, 0);

    RetVal = Builder.CreateInsertElement(RetVal, low, i);

    Value *carry = Builder.CreateExtractValue(lowWithC, 1);

    // Ext i1 to i32

    carry = Builder.CreateZExt(carry, helper.i32Ty);

    Value *hi0 = Builder.CreateExtractElement(op0, i + 1);

    Value *hi1 = Builder.CreateExtractElement(op1, i + 1);

    Value *hi = Builder.CreateAdd(hi0, hi1);

    hi = Builder.CreateAdd(hi, carry);

    RetVal = Builder.CreateInsertElement(RetVal, hi, i + 1);

  }

  return RetVal;

}

bool IsValidLoadInput(Value *V) {

  // Must be load input.

  // TODO: report this error on front-end

  if (!V || !isa<CallInst>(V)) {

    return false;

  }

  CallInst *CI = cast<CallInst>(V);

  // Must be immediate.

  ConstantInt *opArg =

      cast<ConstantInt>(CI->getArgOperand(DXIL::OperandIndex::kOpcodeIdx));

  DXIL::OpCode op = static_cast<DXIL::OpCode>(opArg->getLimitedValue());

  if (op != DXIL::OpCode::LoadInput) {

    return false;

  }

  return true;

}

// Tunnel through insert/extract element and shuffle to find original source

// of scalar value, or specified element (vecIdx) of vector value.

Value *FindScalarSource(Value *src, unsigned vecIdx = 0) {

  Type *srcTy = src->getType()->getScalarType();

  while (src && !isa<UndefValue>(src)) {

    if (src->getType()->isVectorTy()) {

      if (InsertElementInst *IE = dyn_cast<InsertElementInst>(src)) {

        unsigned curIdx = (unsigned)cast<ConstantInt>(IE->getOperand(2))

                              ->getUniqueInteger()

                              .getLimitedValue();

        src = IE->getOperand((curIdx == vecIdx) ? 
1938
 : 
03.25k
);

      } else 
if (ShuffleVectorInst *916
SV916
 = dyn_cast<ShuffleVectorInst>(src)) {

        int newIdx = SV->getMaskValue(vecIdx);

        if (newIdx < 0)

          return UndefValue::get(srcTy);

        vecIdx = (unsigned)newIdx;

        src = SV->getOperand(0);

        unsigned numElt = src->getType()->getVectorNumElements();

        if (numElt <= vecIdx) {

          vecIdx -= numElt;

          src = SV->getOperand(1);

        }

      } else {

        return UndefValue::get(srcTy); // Didn't find it.

      }

    } else {

      if (ExtractElementInst *EE = dyn_cast<ExtractElementInst>(src)) {

        vecIdx = (unsigned)cast<ConstantInt>(EE->getIndexOperand())

                     ->getUniqueInteger()

                     .getLimitedValue();

        src = EE->getVectorOperand();

      } else if (hlsl::dxilutil::IsConvergentMarker(src)) {

        src = hlsl::dxilutil::GetConvergentSource(src);

      } else {

        break; // Found it.

      }

    }

  }

  return src;

}

// Finds corresponding inputs, calls translation for each, and returns

// resulting vector or scalar.

// Uses functor that takes (inputElemID, rowIdx, colIdx), and returns

// translation for one input scalar.

Value *TranslateEvalHelper(

    CallInst *CI, Value *val, IRBuilder<> &Builder,

    std::function<Value *(Value *, Value *, Value *)> fnTranslateScalarInput) {

  Type *Ty = CI->getType();

  Value *result = UndefValue::get(Ty);

  if (Ty->isVectorTy()) {

    for (unsigned i = 0; i < Ty->getVectorNumElements(); 
++i882
) {

      Value *InputEl = FindScalarSource(val, i);

      if (!IsValidLoadInput(InputEl)) {

        dxilutil::EmitErrorOnInstruction(

            CI, "attribute evaluation can only be done "

                "on values taken directly from inputs.");

        return result;

      }

      CallInst *loadInput = cast<CallInst>(InputEl);

      Value *inputElemID =

          loadInput->getArgOperand(DXIL::OperandIndex::kLoadInputIDOpIdx);

      Value *rowIdx =

          loadInput->getArgOperand(DXIL::OperandIndex::kLoadInputRowOpIdx);

      Value *colIdx =

          loadInput->getArgOperand(DXIL::OperandIndex::kLoadInputColOpIdx);

      Value *Elt = fnTranslateScalarInput(inputElemID, rowIdx, colIdx);