/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/

#include "Compiler/Optimizer/OpenCLPasses/ReplaceUnsupportedIntrinsics/ReplaceUnsupportedIntrinsics.hpp"
#include "Compiler/CodeGenPublic.h"
#include "Compiler/IGCPassSupport.h"
#include "common/igc_regkeys.hpp"

#include "common/LLVMWarningsPush.hpp"

#include "llvmWrapper/IR/Instructions.h"

#include <llvm/IR/DerivedTypes.h>
#include <llvm/IR/Module.h>
#include <llvm/IR/Function.h>
#include <llvm/IR/Instructions.h>
#include <llvm/IR/Intrinsics.h>
#include <llvm/IR/IntrinsicInst.h>
#include "common/LLVMWarningsPop.hpp"

using namespace llvm;
using namespace IGC;

// Register pass to igc-opt
#define PASS_FLAG "igc-replace-unsupported-intrinsics"
#define PASS_DESCRIPTION "Replace calls to instrinsics which are not supported by the codegen"
#define PASS_CFG_ONLY false
#define PASS_ANALYSIS false
IGC_INITIALIZE_PASS_BEGIN(ReplaceUnsupportedIntrinsics, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
IGC_INITIALIZE_PASS_END(ReplaceUnsupportedIntrinsics, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)

char ReplaceUnsupportedIntrinsics::ID = 0;

ReplaceUnsupportedIntrinsics::ReplaceUnsupportedIntrinsics() : ModulePass(ID)
{
    initializeReplaceUnsupportedIntrinsicsPass(*PassRegistry::getPassRegistry());
}

// if the value comes from a bitcast, return the source, otherwise return itself
static Value* SkipBitCast(Value* v)
{
    if(BitCastInst* bc = dyn_cast<BitCastInst>(v))
    {
        v = bc->getOperand(0);
    }
    return v;
}

static MemCpyInst *MemMoveToMemCpy(MemMoveInst *MM)
{
    SmallVector<Value*, 5> args;
    for (unsigned i=0; i < MM->getNumArgOperands(); i++)
        args.push_back(MM->getArgOperand(i));

    auto *Dst  = MM->getRawDest();
    auto *Src  = MM->getRawSource();
    auto *Size = MM->getLength();

    Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
    auto *M = MM->getParent()->getParent()->getParent();
    Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);

    return cast<MemCpyInst>(MemCpyInst::Create(TheFn, args));
}

// insertReverseLoop - Insert an empty loop at the end of BB 'Loc'.
// The loop's induction variable iterates from 'Length'-1 to 0.
// The return value is the value of the induction variable in the loop's body.
static Instruction *insertReverseLoop(BasicBlock *Loc, BasicBlock *Post, Value *Length, StringRef BBName)
{
    DebugLoc DL = Loc->getTerminator()->getDebugLoc();
    Function *F = Loc->getParent();
    LLVMContext &C = F->getContext();
    IntegerType* LengthType = cast<IntegerType>(Length->getType());
    // Create an alloca for storing the loop's induction variable
    Value* pIV = new AllocaInst(LengthType, 0, "pIV", &(*F->getEntryBlock().begin()));
    // Split the BB at the location of the call
    BasicBlock *Pre = Loc;
    // Create a new BB for the loop Body
    BasicBlock *Body = BasicBlock::Create(C, Twine(BBName) + ".body", F, Post);
    ConstantInt* Zero = ConstantInt::get(LengthType, 0);
    ConstantInt* One = ConstantInt::get(LengthType, 1);
    {
        // Remove the unconditional 'br' instruction which will be replaced by a conditional 'br'
        Pre->getTerminator()->eraseFromParent();
        IRBuilder<> B(Pre);
        B.SetCurrentDebugLocation(DL);
        // Init the IV            
        auto *Init = B.CreateSub(Length, One);
        B.CreateStore(Init, pIV);
        Value *IsContinue = B.CreateICmpSGE(Init, Zero);
        B.CreateCondBr(IsContinue, Body, Post);
    }
    // The induction variable's value
    Instruction *IV;
    {
        // Loop body's Basic Block
        IRBuilder<> B(Body);
        B.SetCurrentDebugLocation(DL);
        IV = B.CreateLoad(pIV, "IV");
        // User of function will add more instructions at this point ...
        // Decrement the IV and check for end of loop
        Value *Dec = B.CreateSub(IV, One);
        B.CreateStore(Dec, pIV);
        Value *IsContinue = B.CreateICmpSGE(Dec, Zero);
        B.CreateCondBr(IsContinue, Body, Post);
    }
    return IV;
}

// insertLoop - Insert an empty loop before instruction 'Loc'.
// The loop's induction variable iterates from 0 to 'Length'-1.
// The return value is the value of the induction variable in the loop's body.
static Instruction *insertLoop(Instruction *Loc, Value *Length, StringRef BBName)
{
    DebugLoc DL = Loc->getDebugLoc();
    Function *F = Loc->getParent()->getParent();
    LLVMContext &C = F->getContext();
    IntegerType* LengthType = cast<IntegerType>(Length->getType());
    // Create an alloca for storing the loop's induction variable
    Value* pIV = new AllocaInst(LengthType, 0, "pIV", &(*F->getEntryBlock().begin()));        
    // Split the BB at the location of the call
    BasicBlock *Pre = Loc->getParent();
    BasicBlock *Post = Pre->splitBasicBlock(
        BasicBlock::iterator(Loc), Twine(BBName) + ".post");
    // Create a new BB for the loop Body
    BasicBlock *Body = BasicBlock::Create(C, Twine(BBName) + ".body", F, Post);
    {
        // Remove the unconditional 'br' instruction which will be replaced by a conditional 'br'
        Pre->getTerminator()->eraseFromParent();
        IRBuilder<> B(Pre);
        B.SetCurrentDebugLocation(DL);
        ConstantInt* Zero = ConstantInt::get(LengthType, 0);
        // Init the IV            
        B.CreateStore(Zero, pIV);
        Value *IsContinue = B.CreateICmpULT(Zero, Length);
        B.CreateCondBr(IsContinue, Body, Post);
    }
    // The induction variable's value
    Instruction *IV;
    {
        // Loop body's Basic Block
        IRBuilder<> B(Body);
        B.SetCurrentDebugLocation(DL);
        IV = B.CreateLoad(pIV, "IV");
        // User of function will add more instructions at this point ...
        // Increment the IV and check for end of loop
        Value *Inc = B.CreateAdd(IV, ConstantInt::get(LengthType, 1));
        B.CreateStore(Inc, pIV);
        Value *IsContinue = B.CreateICmpULT(Inc, Length);
        B.CreateCondBr(IsContinue, Body, Post);
    }
    return IV;
}

Value *ReplaceUnsupportedIntrinsics::replicateScalar(
    Value *ScalarVal, Type* Ty, Instruction *InsertBefore)
{
    VectorType *VTy = dyn_cast<VectorType>(Ty);
    Type *ETy = VTy ? VTy->getElementType() : Ty;
    uint32_t sBits = ScalarVal->getType()->getPrimitiveSizeInBits();
    uint32_t nBits = ETy->getPrimitiveSizeInBits();
    assert((nBits % sBits) == 0 && nBits <= 64 && "Type mismatch in replicateScalar!");
    uint32_t ratio = nBits/sBits;

    IRBuilder<> Builder(InsertBefore);
    Value *NewVal;
    if(ratio > 1)
    {
        if(ConstantInt *CI = dyn_cast<ConstantInt>(ScalarVal))
        {
            uint64_t s = CI->getZExtValue();
            uint64_t n = s;
            for(unsigned i = 1; i < ratio; ++i)
            {
                n = (n << sBits) | s;
            }
            NewVal = ConstantInt::get(ETy, n);
        }
        else
        {
            Value *nScalarVal = Builder.CreateZExt(ScalarVal, ETy);
            NewVal = nScalarVal;
            for(unsigned i = 1; i < ratio; ++i)
            {
                NewVal = Builder.CreateShl(NewVal, sBits);
                NewVal = Builder.CreateAdd(NewVal, nScalarVal);
            }
        }
    }
    else
    {
        NewVal = ScalarVal;
    }

    Value *Res;
    if( VTy )
    {
        Res = UndefValue::get(VTy);
        Type *TyI32 = Type::getInt32Ty(ScalarVal->getContext());
        for(unsigned i=0; i < VTy->getNumElements(); ++i)
        {
            Value *Idx = ConstantInt::get(TyI32, i);
            Res = Builder.CreateInsertElement(Res, NewVal, Idx);
        }
    }
    else
    {
        Res = NewVal;
    }
    return Res;
}

// A help functions to generate vector load or stores for efficient
// memory operations.
//
// groupI8Stream() groups a stream of i8 into a stream of <8xi32> as
// much as possible. Then for the remaining i8's ( < 32), group them
// into vectors of element type i32 and/or i8. This results in at most
// the following 5 vectors and/or scalars:
//    <4xi32>, <3xi32> or <2xi32>, i32, <2xi8>, i8
// Note that we will not generate <3xi8> (see also the code for details).
// For example, given 127 i8's, we can
// have:
//   <8xi32>, <8xi32>, <8xi32>, <4xi32>, <3xi32>, <2xi8>, i8
//
// The grouping result are kept in Vecs, L (actual length of Vecs),
// and NumI32 (the number of <8xi32>, ie. the number of Vecs[0]. For all
// the other vectors/scalars, ie Vecs[1 : L-1], the number is always 1).
// For the above case, they are:
//    Vecs[0] = <8xi32>
//    Vecs[1] = <4xi32>
//    Vecs[2] = <3xi32>
//    Vecs[3] = <2xi8>
//    Vecs[4] = i8
//    L = 5;
//    NumI32 = 3;
//
// We may generate <3xi32>, but not <3xi8> as <3xi32> can be loaded
// or stored by a single send instruction, where <3xi8> cannot (even
// <3xi8> can be splitted later in VectorProcessing, but it's better
// not generate <3xi8> vector in the first place).

// Note that Vecs[] should be allocated by callers with enough space
// to hold all vectors (6 should be enough; 1 for <8xi32>, 5 for the
// others). 
void ReplaceUnsupportedIntrinsics::groupI8Stream(
    LLVMContext& C, uint32_t NumI8, uint32_t Align,
    uint32_t& NumI32, Type** Vecs, uint32_t& L)
{
    NumI32 = NumI8 / 32; // size of <8xi32> = 32
    uint32_t RemI8 = NumI8 % 32;
    uint32_t CntI32 = RemI8 / 4; // the number of i32
    uint32_t CntI8  = RemI8 % 4; // remaining number of i8(0-3)

    Type *TyI32 = Type::getInt32Ty(C);
    Type *TyI8  = Type::getInt8Ty(C);

    uint32_t n = 0;
    Vecs[n++] = VectorType::get(TyI32, 8);
    // CntI32 range [0, 7]
    if(CntI32 >= 4)
    {
        Vecs[n++] = VectorType::get(TyI32, 4);
        CntI32 -= 4;
    }
    if(CntI32 == 3 && Align >= 4)
    {
        Vecs[n++] = VectorType::get(TyI32, 3);
        CntI32 -= 3;
    }
    if(CntI32 >= 2)
    {
        Vecs[n++] = VectorType::get(TyI32, 2);
        CntI32 -= 2;
    }
    if (CntI32 > 0)
    {
        Vecs[n++] = TyI32;
        CntI32 -= 1;
    }
    assert(CntI32 == 0 && "Did not handle all types of i32");

    // CntI8 range [0, 3]
    if(CntI8 >= 2)
    {
        Vecs[n++] = VectorType::get(TyI8, 2);
        CntI8 -= 2;
    }
    if(CntI8 > 0)
    {
        Vecs[n++] = TyI8;
        CntI8 -= 1;
    }
    assert(CntI8 == 0 && "Did not handle all types of i8");

    L = n;
}

void ReplaceUnsupportedIntrinsics::replaceMemcpy(MemCpyInst* MC)
{
    // The idea is to convert
    //
    //   memcpy (i8* Dst, i8* Src, len)
    //
    // into a vector load and store for cases where "len" is
    // constant. If "len" isn't constant,  just use i8 copy as
    // this should not happen with OCL code (all memcpy is 
    // generated by the compiler for cases such as structure
    // assignment, etc.)
    //
    // If len is constant, it will be transferred to
    //
    //   lenv8 = len / 32 (<8xi32>);
    //   len_rem = len % 32;
    //
    //   // main loop
    //   dstV8 = bitcast Dst, <8xi32>*
    //   srcV8 = bitcast Src, <8xi32>*
    //   for(i=0; i < lenv8; ++i)
    //     dstV8[i] = srcV8[i];
    //
    //   // epilog, process remaining elements
    //   for(i=0; i < len_rem; ++i)
    //     Dst[lenv8*32 + i] = Src[lenv8*32 + i];
    //
    //   Note that the above epilog loop is optimized away with
    //   as much as possible <nxi32> and <mxi8> loads and stores.
    //
    // Selecting 8 as vector length is due to that A64 messages can
    // load eight i32 per SIMD channel. A32 will have 2 loads/stores
    // for each eight i32, which is still efficient. Unaligned vector
    // will be handled correctly and effciently later in vector load
    // and store emit.
    Value *Dst = MC->getRawDest();
    Value *Src = MC->getRawSource();
    Value *LPCount = MC->getLength();
    uint32_t Align = IGCLLVM::getDestAlignment(MC);
    const bool IsVolatile = MC->isVolatile();
    const uint32_t SrcAS = MC->getSourceAddressSpace();
    const uint32_t DstAS = MC->getDestAddressSpace();

    LLVMContext& C = MC->getContext();
    Type *TySrcPtrI8 = Type::getInt8PtrTy(C, SrcAS);
    Type *TyDstPtrI8 = Type::getInt8PtrTy(C, DstAS);

    IRBuilder<> Builder(MC);

    ConstantInt *CI = dyn_cast<ConstantInt>(LPCount);
    if(CI)
    {
        uint32_t Count = (uint32_t)CI->getZExtValue();

        Type *VecTys[8];
        uint32_t Len, NewCount;
        groupI8Stream(C, Count, Align, NewCount, VecTys, Len);

        Value *NewSrc, *NewDst, *vDst, *vSrc;
        uint32_t BOfst = 0; // Byte offset

        // First, insert main loop before MC.
        // Note that if NewCount is small, we may directly generate ld/st
        // without generating the loop. 
        if( NewCount > 0 )
        {
            vSrc = Builder.CreateBitCast(SkipBitCast(Src), PointerType::get(VecTys[0], SrcAS), "memcpy_vsrc");
            vDst = Builder.CreateBitCast(SkipBitCast(Dst), PointerType::get(VecTys[0], DstAS), "memcpy_vdst");

            // getPrimitiveSizeInBits() should be enough, no need to
            // use DataLayout to get target-dependent size.
            uint32_t SZ = (VecTys[0]->getPrimitiveSizeInBits() / 8);

            // To set alignment correctly
            uint32_t adjust_align = getLargestPowerOfTwo(SZ);
            Align = adjust_align < Align ? adjust_align : Align;

            // If NewCount is less than 6,  don't generate loop.
            // Note that 6 is just an arbitrary number here.
            if(NewCount < 6)
            {
                for(unsigned i=0; i < NewCount; ++i)
                {
                    Value *tSrc = Builder.CreateConstGEP1_32(vSrc, i);
                    Value *tDst = Builder.CreateConstGEP1_32(vDst, i);
                    LoadInst *L = Builder.CreateAlignedLoad(tSrc, Align, IsVolatile); 
                    (void) Builder.CreateAlignedStore(L, tDst, Align, IsVolatile);
                }
            }
            else
            {
                Value *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
                Instruction *IV = insertLoop(MC, NewLPCount, "memcpy");
                {
                    IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
                    Value *tSrc = B.CreateGEP(vSrc, IV);
                    Value *tDst = B.CreateGEP(vDst, IV);
                    LoadInst *L = B.CreateAlignedLoad(tSrc, Align, IsVolatile);
                    (void) B.CreateAlignedStore(L, tDst, Align, IsVolatile);
                }
            }
            
            BOfst = NewCount * SZ;
        }

        // Second, generate epilog code before MC.
        // Note that as MC has been moved to a different BB by
        //   inserting the main loop! Reset it to MC.
        Builder.SetInsertPoint(MC);
        if (Len > 1)
        {
            Src = Builder.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
            Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");
        }
        for(unsigned i=1; i < Len; ++i)
        {
            uint32_t SZ = VecTys[i]->getPrimitiveSizeInBits() / 8;
            uint32_t adjust_align = getLargestPowerOfTwo(SZ);
            Align = adjust_align < Align ? adjust_align : Align;
            NewSrc = BOfst > 0 ? Builder.CreateConstGEP1_32(Src, BOfst) : Src;
            NewDst = BOfst > 0 ? Builder.CreateConstGEP1_32(Dst, BOfst) : Dst;
            vSrc = Builder.CreateBitCast(SkipBitCast(NewSrc), PointerType::get(VecTys[i], SrcAS), "memcpy_rem");
            vDst = Builder.CreateBitCast(SkipBitCast(NewDst), PointerType::get(VecTys[i], DstAS), "memcpy_rem");
            LoadInst *L = Builder.CreateAlignedLoad(vSrc, Align, IsVolatile);
            (void) Builder.CreateAlignedStore(L, vDst, Align, IsVolatile);
            BOfst += SZ;
        }
    }
    else
    {
        Src = Builder.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
        Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");
        // Fall back to i8 copy
        Instruction *IV = insertLoop(MC, LPCount, "memcpy");
        {
            IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
            Value *tSrc = B.CreateGEP(Src, IV);
            Value *tDst = B.CreateGEP(Dst, IV);
            LoadInst *L = B.CreateAlignedLoad(tSrc, Align, IsVolatile);
            (void) B.CreateAlignedStore(L, tDst, Align, IsVolatile);
        }
    }
    MC->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceMemMove(MemMoveInst* MM)
{
    Value *Dst = MM->getRawDest();
    Value *Src = MM->getRawSource();
    Value *LPCount = MM->getLength();
    uint32_t Align = IGCLLVM::getDestAlignment(MM);
    if (Align == 0)
        Align = 1;
    const bool IsVolatile = MM->isVolatile();
    const uint32_t SrcAS = MM->getSourceAddressSpace();
    const uint32_t DstAS = MM->getDestAddressSpace();

    // If non-generic address spaces mismatch, they can't alias
    // and we can do a memcpy.

    if (SrcAS < ADDRESS_SPACE_NUM_ADDRESSES &&
        DstAS < ADDRESS_SPACE_NUM_ADDRESSES &&
        SrcAS != ADDRESS_SPACE_GENERIC &&
        DstAS != ADDRESS_SPACE_GENERIC &&
        SrcAS != DstAS)
    {
        auto *MemCpy = MemMoveToMemCpy(MM);
        MemCpy->insertBefore(MM);
        replaceMemcpy(MemCpy);
        MM->eraseFromParent();
        return;
    }

    LLVMContext& C = MM->getContext();
    Type *TySrcPtrI8 = Type::getInt8PtrTy(C, SrcAS);
    Type *TyDstPtrI8 = Type::getInt8PtrTy(C, DstAS);

    auto *F = MM->getParent()->getParent();

    IRBuilder<> B(MM);

    auto *i8Src = B.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
    auto *i8Dst = B.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");

    // Setup control flow to do:
    // if (Src < Dst)
    //   reverse copy data
    // else
    //   normal copy (such as memcpy())

    // Src < Dst
    Value *pCmp = nullptr;
    {
        auto *cmpCastSrc = (DstAS == ADDRESS_SPACE_GENERIC) ?
            B.CreateAddrSpaceCast(i8Src, TyDstPtrI8) : i8Src;
        auto *cmpCastDst = (SrcAS == ADDRESS_SPACE_GENERIC) ?
            B.CreateAddrSpaceCast(i8Dst, TySrcPtrI8) : i8Dst;

        pCmp = B.CreateICmpULT(cmpCastSrc, cmpCastDst);
    }

    auto *Pre = MM->getParent();
    auto *Post = Pre->splitBasicBlock(MM, "memmove.post");

    Pre->getTerminator()->eraseFromParent();

    auto *BBTrue = BasicBlock::Create(C, "memmove.true", F, Post);
    auto *BBFalse = BasicBlock::Create(C, "memmove.false", F, Post);

    B.SetInsertPoint(Pre);
    B.CreateCondBr(pCmp, BBTrue, BBFalse);

    B.SetInsertPoint(BBTrue);
    B.CreateBr(Post);
    B.SetInsertPoint(BBFalse);
    B.CreateBr(Post);

    auto *CI = dyn_cast<ConstantInt>(LPCount);
    if (CI)
    {
        uint32_t Count = (uint32_t)CI->getZExtValue();

        // noop
        if (Count == 0)
        {
            MM->eraseFromParent();
            return;
        }

        Type *VecTys[8];
        uint32_t Len, NewCount;
        groupI8Stream(C, Count, Align, NewCount, VecTys, Len);

        // for true block (Src < Dst), do a reverse copy.
        {
            B.SetInsertPoint(BBTrue->getTerminator());

            // calculate byte offsets so we can walk backwards through them
            SmallVector<uint, 8> byteOffsets { 0 };

            {
                uint32_t SZ = (VecTys[0]->getPrimitiveSizeInBits() / 8);
                uint32_t BOfst = NewCount * SZ;

                for (unsigned i = 1; i < Len; i++)
                {
                    byteOffsets.push_back(BOfst);
                    uint32_t SZ = (VecTys[i]->getPrimitiveSizeInBits() / 8);
                    BOfst += SZ;
                }
            }

            // emit the smaller than <8 x i32> stores
            for (unsigned i=Len - 1; i >= 1; i--)
            {
                uint offset = byteOffsets[i];
                uint32_t newAlign = getLargestPowerOfTwo(Align + offset);
                auto *tSrc = B.CreateConstGEP1_32(i8Src, offset);
                auto *tDst = B.CreateConstGEP1_32(i8Dst, offset);

                auto *vSrc = B.CreateBitCast(SkipBitCast(tSrc), PointerType::get(VecTys[i], SrcAS), "memcpy_rem");
                auto *vDst = B.CreateBitCast(SkipBitCast(tDst), PointerType::get(VecTys[i], DstAS), "memcpy_rem");
                LoadInst *L = B.CreateAlignedLoad(vSrc, newAlign, IsVolatile);
                (void)B.CreateAlignedStore(L, vDst, newAlign, IsVolatile);
            }

            // now emit the <8 x i32> stores
            auto *vSrc = B.CreateBitCast(SkipBitCast(Src), PointerType::get(VecTys[0], SrcAS), "memcpy_vsrc");
            auto *vDst = B.CreateBitCast(SkipBitCast(Dst), PointerType::get(VecTys[0], DstAS), "memcpy_vdst");
            // If NewCount is less than 6,  don't generate loop.
            // Note that 6 is just an arbitrary number here.
            uint32_t SZ = (VecTys[0]->getPrimitiveSizeInBits() / 8);
            uint32_t newAlign = getLargestPowerOfTwo(Align + SZ);
            if (NewCount < 6)
            {
                for (unsigned i = 0; i < NewCount; i++)
                {
                    unsigned idx = NewCount - 1 - i;
                    auto *tSrc = B.CreateConstGEP1_32(vSrc, idx);
                    auto *tDst = B.CreateConstGEP1_32(vDst, idx);
                    LoadInst *L = B.CreateAlignedLoad(tSrc, newAlign, IsVolatile);
                    (void)B.CreateAlignedStore(L, tDst, newAlign, IsVolatile);
                }
            }
            else
            {
                auto *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
                Instruction *IV = insertReverseLoop(BBTrue, Post, NewLPCount, "memmmove");
                {
                    IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
                    Value *tSrc = B.CreateGEP(vSrc, IV);
                    Value *tDst = B.CreateGEP(vDst, IV);
                    LoadInst *L = B.CreateAlignedLoad(tSrc, newAlign, IsVolatile);
                    (void)B.CreateAlignedStore(L, tDst, newAlign, IsVolatile);
                }
            }
        }

        // for false block (Src >= Dst), just a plain memcpy.
        {
            auto *MemCpy = MemMoveToMemCpy(MM);
            MemCpy->insertBefore(BBFalse->getTerminator());
            replaceMemcpy(MemCpy);
        }
    }
    else
    {
        // (Src < Dst)
        {
            B.SetInsertPoint(BBTrue->getTerminator());
            // Fall back to i8 copy
            Instruction *IV = insertReverseLoop(BBTrue, Post, LPCount, "memmove");
            {
                IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
                Value *tSrc = B.CreateGEP(i8Src, IV);
                Value *tDst = B.CreateGEP(i8Dst, IV);
                LoadInst *L = B.CreateAlignedLoad(tSrc, 1, IsVolatile);
                (void)B.CreateAlignedStore(L, tDst, 1, IsVolatile);
            }
        }

        // for false block (Src >= Dst), just a plain memcpy.
        {
            auto *MemCpy = MemMoveToMemCpy(MM);
            MemCpy->insertBefore(BBFalse->getTerminator());
            replaceMemcpy(MemCpy);
        }
    }

    MM->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceMemset(MemSetInst* MS)
{
    // Same idea as replaceMemcpy (see comment of replaceMemcpy). 
    Value *Dst = MS->getRawDest();
    Value *Src = MS->getValue();
    Value *LPCount = MS->getLength();
    uint32_t Align = IGCLLVM::getDestAlignment(MS);
    const bool IsVolatile = MS->isVolatile();
    const uint32_t AS = MS->getDestAddressSpace();

    LLVMContext& C = MS->getContext();
    Type *TyPtrI8 = Type::getInt8PtrTy(C, AS);

    IRBuilder<> Builder(MS);

    ConstantInt *CI = dyn_cast<ConstantInt>(LPCount);
    if(CI)
    {
        uint32_t Count = (uint32_t)CI->getZExtValue();

        Type *VecTys[8];
        uint32_t Len, NewCount;
        groupI8Stream(C, Count, Align, NewCount, VecTys, Len);

        Value *NewDst, *vDst, *vSrc;
        uint32_t BOfst = 0; // Byte offset

        // First, insert main loop before MC.
        if( NewCount > 0 )
        {
            PointerType* PTy = PointerType::get(VecTys[0], AS);
            vSrc = replicateScalar(Src, VecTys[0], MS);
            vDst = Builder.CreateBitCast(SkipBitCast(Dst), PTy, "memset_vdst");

            // getPrimitiveSizeInBits() should be enough, no need to
            // use DataLayout to get target-dependent size.
            uint32_t SZ = (VecTys[0]->getPrimitiveSizeInBits() / 8);

            // To set alignment correctly
            uint32_t adjust_align = getLargestPowerOfTwo(SZ);
            Align = adjust_align < Align ? adjust_align : Align;

            // If NewCount is less than 6,  don't generate loop.
            // Note that 6 is just an arbitrary number here.
            if(NewCount < 6)
            {
                for(unsigned i=0; i < NewCount; ++i)
                {
                    Value *tDst = Builder.CreateConstGEP1_32(vDst, i);
                    (void) Builder.CreateAlignedStore(vSrc, tDst, Align, IsVolatile);
                }
            }
            else
            {
                Value *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
                Instruction *IV = insertLoop(MS, NewLPCount, "memset");
                {
                    IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
                    Value *tDst = B.CreateGEP(vDst, IV);
                    (void) B.CreateAlignedStore(vSrc, tDst, Align, IsVolatile);
                }
            }
 
            // Set offset for the remaining elements
            BOfst = NewCount * SZ; 
        }

        // Second, generate epilog code before MS.
        // Note that as MC has been moved to a different BB by
        //   inserting the main loop! Reset it to MS.
        Builder.SetInsertPoint(MS);
        if (Len > 1)
        {
            Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyPtrI8, "memset_dst");
        }
        for(unsigned i=1; i < Len; ++i)
        {
            uint32_t SZ = VecTys[i]->getPrimitiveSizeInBits() / 8;
            uint32_t adjust_align = getLargestPowerOfTwo(SZ);
            Align = adjust_align < Align ? adjust_align : Align;
            PointerType *PTy = PointerType::get(VecTys[i], AS);
            NewDst = BOfst > 0 ? Builder.CreateConstGEP1_32(Dst, BOfst) : Dst;
            vSrc = replicateScalar(Src, VecTys[i], MS);
            vDst = Builder.CreateBitCast(SkipBitCast(NewDst), PTy, "memset_rem");
            (void) Builder.CreateAlignedStore(vSrc, vDst, Align, IsVolatile);
            BOfst += SZ;
        }
    }
    else
    {
        Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyPtrI8, "memset_dst");
        // Fall back to i8 copy
        Instruction *IV = insertLoop(MS, LPCount, "memset");
        {
            IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
            Value *tDst = B.CreateGEP(Dst, IV);
            (void) B.CreateAlignedStore(Src, tDst, Align, IsVolatile);
        }
    }
    MS->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceExpect(IntrinsicInst* MS)
{
    MS->replaceAllUsesWith(MS->getOperand(0));
    MS->eraseFromParent();
}

bool ReplaceUnsupportedIntrinsics::runOnModule(Module &M)
{
    // Calls to the memset intrinsics which will be replaced
    std::vector<MemSetInst*> memsetInsts;
    // Calls to the memcpy's
    std::vector<MemCpyInst*> memcpyInsts;
    // Calls to the memmove's
    std::vector<MemMoveInst*> memMoveInsts;
    // Calls to the expect intrinsics which will be replaced
    std::vector<IntrinsicInst*> expectInsts;

    // Look for declarations of intrinsics of interest and their call users
    for (auto I = M.begin(), IE = M.end(); I != IE; ++I)
    {
        Function &F = *I;
        if (!F.isIntrinsic()) continue;
        // llvm.memset.*
        if (F.getIntrinsicID() == Intrinsic::memset)
        {
            for (auto UI = F.user_begin(), UE = F.user_end(); UI != UE; ++UI)
            {
                if (MemSetInst* MS = dyn_cast<MemSetInst>(*UI))
                {
                    memsetInsts.push_back(MS);
                }
            }
        }
        // llvm.memcpy.*
        if (F.getIntrinsicID() == Intrinsic::memcpy)
        {
            for (auto UI = F.user_begin(), UE = F.user_end(); UI != UE; ++UI)
            {
                if (MemCpyInst* MC = dyn_cast<MemCpyInst>(*UI))
                {
                    memcpyInsts.push_back(MC);
                }
            }
        }
        // llvm.memmove.*
        if (F.getIntrinsicID() == Intrinsic::memmove)
        {
            for (auto UI = F.user_begin(), UE = F.user_end(); UI != UE; ++UI)
            {
                if (auto * MM = dyn_cast<MemMoveInst>(*UI))
                {
                    memMoveInsts.push_back(MM);
                }
            }
        }
        // llvm.expect.*
        if (F.getIntrinsicID() == Intrinsic::expect)
        {
            for (auto UI = F.user_begin(), UE = F.user_end(); UI != UE; ++UI)
            {
                if (IntrinsicInst* EX = dyn_cast<IntrinsicInst>(*UI))
                {
                    expectInsts.push_back(EX);
                }
            }
        }
    }
    // Replace calls to memset with equivalent loops
    for (auto I : memsetInsts)
    {
        MemSetInst& MS = *I;
        replaceMemset(&MS);
    }
    // Replace calls to memcpy with equivalent loops
    for (auto I : memcpyInsts)
    {
        MemCpyInst& MC = *I;
        replaceMemcpy(&MC);
    }
    // Replace calls to memmove with equivalent loops
    for (auto I : memMoveInsts)
    {
        MemMoveInst& MM = *I;
        replaceMemMove(&MM);
    }
    for (auto I : expectInsts)
    {
        IntrinsicInst& EX = *I;
        replaceExpect(&EX);
    }
    // If there were memset, memmove, memcpy or expect instrinsic then module was changed.
    return !memsetInsts.empty() || !memcpyInsts.empty() || !expectInsts.empty() || !memMoveInsts.empty();
}
