/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/

//===-  IBiF_VME_VA.cl - VME, VA helper functions               -===//
//
// This file defines helper builtins of OpenCL VME, VA extension functions.
//
//===----------------------------------------------------------------------===//


/*****************************************************************************/
/*                       VME (Vide Motion Estimation)                        */
/*****************************************************************************/

INLINE uint OVERLOADABLE intel_get_accelerator_mb_block_type( sampler_t accelerator )
{
   return __builtin_IB_vme_mb_block_type();
}

INLINE uint OVERLOADABLE intel_get_accelerator_mb_sub_pixel_mode( sampler_t accelerator )
{
   return __builtin_IB_vme_subpixel_mode();
}

INLINE uint OVERLOADABLE intel_get_accelerator_mb_sad_sdjust_mode( sampler_t accelerator )
{
   return __builtin_IB_vme_sad_adjust_mode();
}

INLINE uint OVERLOADABLE intel_get_accelerator_mb_search_path_type( sampler_t accelerator )
{
   return __builtin_IB_vme_search_path_type();
}

#ifdef __IGC_BUILD__
// TODO: Should we include this header somehow?
// These defines should be syncronizes with the values of the following enums
// defined in mainline\Source\USC\state\SamplerTypes.h
// USC::SAMPLER_VME_MB_BLOCK_TYPE
// USC::SAMPLER_VME_SUBPIXEL_MODE
// USC::SAMPLER_VME_SAD_ADJUST_MODE
// USC::SAMPLER_VME_SEARCH_PATH_TYPE

#define SAMPLER_VME_MB_BLOCK_TYPE_16x16     0x0
#define SAMPLER_VME_MB_BLOCK_TYPE_8x8       0x1
#define SAMPLER_VME_MB_BLOCK_TYPE_4x4       0x2

#define SAMPLER_VME_SUBPIXEL_MODE_INTEGER   0x0
#define SAMPLER_VME_SUBPIXEL_MODE_HPEL      0x1
#define SAMPLER_VME_SUBPIXEL_MODE_QPEL      0x2

#define SAMPLER_VME_SAD_ADJUST_MODE_NONE    0x0
#define SAMPLER_VME_SAD_ADJUST_MODE_HAAR    0x1

#define SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_2_2     0x0
#define SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_4_4     0x1
// 0x2 - Reserved for the SMALL search path
// 0x3 - Reserved for the SMALL_DIAMOND search path
// 0x4 - Reserved for the LARGE_DIAMOND search path
#define SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_16_12    0x5

#define EU_VME_SHAPEMASK_4x4            0x3F
#define EU_VME_SHAPEMASK_8x8            0x77
#define EU_VME_SHAPEMASK_16x16          0x7E

#define EU_VME_SADADJUST_MODE_NONE      0
#define EU_VME_SADADJUST_MODE_HAAR      2

#define EU_VME_SUBPEL_MODE_INT_PEL      0x0
#define EU_VME_SUBPEL_MODE_H_PEL        0x1
#define EU_VME_SUBPEL_MODE_Q_PEL        0x3

#define UNIVERSAL_INPUT_MESSAGE_NUM_GRFS 4
#define INPUT_MESSAGE_SIC_NUM_GRFS       4
#define RETURN_MESSAGE_NUM_GRFS          7

#define REF0_SKIP_CENTER_0    0x01
#define REF0_ALL_SKIP_CENTERS 0x55

#define HAAR_TRANSFORM_ADJUSTED 0x2

#define INTRA_COMPUTE_TYPE_DISABLED  0x02
#define INTRA_COMPUTE_TYPE_LUMA_ONLY 0x01

#define NUM_DWORD_IN_GRF 8

enum {
    VME_STREAM_DISABLE = 0,
    VME_STREAM_OUT     = 1,
    VME_STREAM_IN      = 2,
    VME_STREAM_INOUT   = 3
};

enum {
    VME_MAJOR_16x16 = 0,
    VME_MAJOR_16x8  = 1,
    VME_MAJOR_8x16  = 2,
    VME_MAJOR_8x8   = 3
};

void create_universal_input_message(GRFHandle   universalInputMsg,
                                    __private ushort2* ref0Coord,
                                    __private ushort2* ref1Coord,
                                    int2               srcCoord,
                                    int2               refCoord) {

  uint M00 = 0;
  uint M01 = 0;
  uint M02 = __builtin_IB_get_message_phase_dw(universalInputMsg, 0, 2);
  uint M03 = __builtin_IB_get_message_phase_dw(universalInputMsg, 0, 3);
  uint M05 = __builtin_IB_get_message_phase_dw(universalInputMsg, 0, 5);
  uint M12 = __builtin_IB_get_message_phase_dw(universalInputMsg, 1, 2);

  // Setup MaxNumMVs - low byte of m1.1.
  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 1, 1, 0x00000010);

  uint blockType = __builtin_IB_vme_mb_block_type();
  if (blockType == SAMPLER_VME_MB_BLOCK_TYPE_4x4) {
      M03 |= 0x3f000000;
  }
  if (blockType == SAMPLER_VME_MB_BLOCK_TYPE_8x8) {
      M03 |= 0x77000000;
  }
  if (blockType == SAMPLER_VME_MB_BLOCK_TYPE_16x16) {
      M03 |= 0x7e000000;
  }

  uint subpixelMode = __builtin_IB_vme_subpixel_mode();
  if (subpixelMode == SAMPLER_VME_SUBPIXEL_MODE_HPEL) {
      M03 |= 0x00001000;
  }
  if (subpixelMode == SAMPLER_VME_SUBPIXEL_MODE_QPEL) {
      M03 |= 0x00003000;
  }

  uint sadAdjustMode = __builtin_IB_vme_sad_adjust_mode();
  if (sadAdjustMode == SAMPLER_VME_SAD_ADJUST_MODE_HAAR) {
      M03 |= 0x00200000;
  }

  uint searchPathType = __builtin_IB_vme_search_path_type();

  if (searchPathType == SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_2_2) {
      M00 |= 0xfffefffe;
      M01 |= 0xfffefffe;
      M05 |= 0x14140000;
      M12 |= 0x00000101;
  }
  if (searchPathType == SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_4_4) {
      M00 |= 0xfffcfffc;
      M01 |= 0xfffcfffc;
      M05 |= 0x18180000;
      M12 |= 0x00000404;
  }
  if (searchPathType == SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_16_12) {
      M00 |= 0xfff4fff0;
      M01 |= 0xfff4fff0;
      M05 |= 0x28300000;
      M12 |= 0x00003030;
  }

  // Set srcX and srcY in the message payload
  M02 += as_uint(convert_ushort2(srcCoord));

  // Set refX and refY in the message payload
  // refCoords (M0.0, M0.1) will set by the IME send message
  *ref0Coord = as_ushort2(M00) + convert_ushort2(refCoord);
  *ref1Coord = as_ushort2(M01);

  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 0, 2, M02);
  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 0, 3, M03);
  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 0, 5, M05);
  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 1, 2, M12);
}

void create_ime_message(GRFHandle imeMsg) {

  // Program the TINY search path by default
  __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 0, 0x000f1001);

  // If we are NOT the TINY search path, program the DEFAULT search path
  uint searchPathType = __builtin_IB_vme_search_path_type();
  if (searchPathType != SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_4_4) {
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 0, 0x01010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 1, 0x10010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 2, 0x0f0f0f0f);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 3, 0x100f0f0f);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 4, 0x01010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 5, 0x10010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 6, 0x0f0f0f0f);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 0, 7, 0x100f0f0f);

      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 1, 0, 0x01010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 1, 1, 0x10010101);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 1, 2, 0x0f0f0f0f);
      __builtin_IB_set_message_phase_legacy_dw(imeMsg, 1, 3, 0x000f0f0f);
  }
}

void create_fbr_message(GRFHandle universalInputMsg, GRFHandle fbrMsg, GRFHandle imeRes) {

  // Disable BME
  uint M03 = __builtin_IB_get_message_phase_dw(universalInputMsg, 0, 3);
  M03 |= 0x00040000;
  __builtin_IB_set_message_phase_legacy_dw(universalInputMsg, 0, 3, M03);

  // Copy result of IME into FBR header
  //fbrMsg[0] = imeRes[1];
  //fbrMsg[1] = imeRes[2];
  //fbrMsg[2] = imeRes[3];
  //fbrMsg[3] = imeRes[4];
  __builtin_IB_set_message_phase_legacy(fbrMsg, 0, __builtin_IB_get_message_phase(imeRes, 1));
  __builtin_IB_set_message_phase_legacy(fbrMsg, 1, __builtin_IB_get_message_phase(imeRes, 2));
  __builtin_IB_set_message_phase_legacy(fbrMsg, 2, __builtin_IB_get_message_phase(imeRes, 3));
  __builtin_IB_set_message_phase_legacy(fbrMsg, 3, __builtin_IB_get_message_phase(imeRes, 4));

}

#endif // __IGC_BUILD__


INLINE void OVERLOADABLE intel_work_group_vme_mb_query(
    __local uint* dst,
    int2      srcCoord,
    int2      refCoord,
    __read_only image2d_t srcImage,
    __read_only image2d_t refImage,
    sampler_t vmeAccelerator )
{
   GRFHandle res = __builtin_IB_create_message_phases(7);

   ushort2 ref0Coord = 0;
   ushort2 ref1Coord = 0;
   GRFHandle universalInputMsg = __builtin_IB_create_message_phases(4);

   create_universal_input_message(universalInputMsg, &ref0Coord, &ref1Coord, srcCoord, refCoord);

   GRFHandle imeMsg = __builtin_IB_create_message_phases(2);
   create_ime_message(imeMsg);
   GRFHandle zeroCenter = __builtin_IB_create_message_phases(1);
   long srcImageInt = (long) __builtin_astype(srcImage, void*);
   long refImageInt = (long) __builtin_astype(refImage, void*);
   __builtin_IB_vme_send_ime(res, universalInputMsg, imeMsg, srcImageInt, refImageInt, as_uint(ref0Coord), as_uint(ref1Coord), zeroCenter);

   if (__builtin_IB_vme_subpixel_mode() != SAMPLER_VME_SUBPIXEL_MODE_INTEGER) {
       GRFHandle fbrMsg = __builtin_IB_create_message_phases(4);
       create_fbr_message(universalInputMsg, fbrMsg, res);

       uint W00                   = __builtin_IB_get_message_phase_dw(res, 0, 0);
       uint W06                   = __builtin_IB_get_message_phase_dw(res, 0, 6);
       __private uchar* W00Bytes  = (__private uchar*)&W00;
       __private uchar* W06Bytes  = (__private uchar*)&W06;

       uint interMbMode        = W00Bytes[0];
       uint subMbShape         = W06Bytes[1];
       // No support for FBRSubPredModeVar yet, create new zeroed var
       uint subMbPredMode     = 0;
       __builtin_IB_vme_send_fbr(res, universalInputMsg, fbrMsg, srcImageInt, refImageInt, interMbMode, subMbShape, subMbPredMode);
   }

   __local uint* tmpDst = dst + __builtin_IB_simd_lane_id();
   tmpDst[0]     = __builtin_IB_get_message_phase(res, 0);
   tmpDst[8]     = __builtin_IB_get_message_phase(res, 1);
   tmpDst[16]    = __builtin_IB_get_message_phase(res, 2);
   tmpDst[24]    = __builtin_IB_get_message_phase(res, 3);
   tmpDst[32]    = __builtin_IB_get_message_phase(res, 4);
   tmpDst[40]    = __builtin_IB_get_message_phase(res, 5);
   // TODO : do we need this copy??? did not appear in original USC code
   tmpDst[48]    = __builtin_IB_get_message_phase(res, 6);
}

#ifdef __IGC_BUILD__
INLINE void OVERLOADABLE DoMultiQuery(
    __local uint* dstSearch,
    uint countPredMVs,
    uint MVCostPrecision,
    uint2 searchCostTable,
    int2 srcCoord,
    int2 predMV,
    image2d_t srcImg,
    image2d_t refImg,
    sampler_t accelerator,
    unsigned MaxPredMVs) // either 4 or 8
{
    GRFHandle VMEUniversal  = __builtin_IB_create_message_phases(UNIVERSAL_INPUT_MESSAGE_NUM_GRFS);
    // Setup MaxNumMVs - M1.1 (bits 5:0).
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 1, 1, 16 /*MaxNumMVs*/);

    // Controls based on mb_block_type
    int dw3_SubMacroblockSubPartitionMask = 0x00000000;
    switch( __builtin_IB_vme_mb_block_type() )
    {
        case SAMPLER_VME_MB_BLOCK_TYPE_16x16:
            dw3_SubMacroblockSubPartitionMask = EU_VME_SHAPEMASK_16x16;
            break;
        case SAMPLER_VME_MB_BLOCK_TYPE_8x8:
            dw3_SubMacroblockSubPartitionMask = EU_VME_SHAPEMASK_8x8;
            break;
        case SAMPLER_VME_MB_BLOCK_TYPE_4x4:
            dw3_SubMacroblockSubPartitionMask = EU_VME_SHAPEMASK_4x4;
            break;
    }
    // Sub-Macroblock Sub-Partition Mask (SubMbPartMask) - M0.3 (bits 30:24)
    dw3_SubMacroblockSubPartitionMask <<= 24;
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 3, dw3_SubMacroblockSubPartitionMask);

    // Controls based on subpixel mode
    int dw3_SubPelMode = 0x00000000;
    switch( __builtin_IB_vme_subpixel_mode() )
    {
        case SAMPLER_VME_SUBPIXEL_MODE_INTEGER:
            dw3_SubPelMode = EU_VME_SUBPEL_MODE_INT_PEL;
            break;
        case SAMPLER_VME_SUBPIXEL_MODE_HPEL:
            dw3_SubPelMode = EU_VME_SUBPEL_MODE_H_PEL;
            break;
        case SAMPLER_VME_SUBPIXEL_MODE_QPEL:
            dw3_SubPelMode = EU_VME_SUBPEL_MODE_Q_PEL;
            break;
    }
    // Sub-Pel Mode (SubPelMode) - M0.3 (bits 13:12)
    dw3_SubPelMode <<= 12;
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 3, dw3_SubPelMode | __builtin_IB_get_message_phase_dw(VMEUniversal, 0, 3) );

    // Controls based on sad adjust mode
    int dw3_InterSADMeasureAdjustment = 0x00000000;
    switch( __builtin_IB_vme_sad_adjust_mode() )
    {
        case SAMPLER_VME_SAD_ADJUST_MODE_NONE:
            dw3_InterSADMeasureAdjustment = EU_VME_SADADJUST_MODE_NONE;
            break;
        case SAMPLER_VME_SAD_ADJUST_MODE_HAAR:
            dw3_InterSADMeasureAdjustment = EU_VME_SADADJUST_MODE_HAAR;
            break;
    }
    // Inter SAD Measure Adjustment (InterSAD) - M0.3 (bits 21:20)
    dw3_InterSADMeasureAdjustment <<= 20;
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 3, dw3_InterSADMeasureAdjustment | __builtin_IB_get_message_phase_dw(VMEUniversal, 0, 3) );

    // Controls based on search path type
    int dw01_refOffset = 0x00000000;
    int dw05_refWindow = 0x00000000;
    int dw12_MaxFixedSearchPathLength_MaximumSearchPathLength = 0x00000000;
    switch( __builtin_IB_vme_search_path_type() )
    {
        case SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_2_2:
            dw01_refOffset = 0xFFFEFFFE; // -2, -2
            dw05_refWindow = 0x14140000; // 20, 20
            dw12_MaxFixedSearchPathLength_MaximumSearchPathLength = 0x00000101;
            break;
        case SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_4_4:
            dw01_refOffset = 0xFFFCFFFC; // -4, -4
            dw05_refWindow = 0x18180000; // 24, 24
            dw12_MaxFixedSearchPathLength_MaximumSearchPathLength = 0x00000404;
            break;
        case SAMPLER_VME_SEARCH_PATH_TYPE_RADIUS_16_12:
            dw01_refOffset = 0xFFF4FFF0; // -16, -12
            dw05_refWindow = 0x28300000; // 48, 40
            dw12_MaxFixedSearchPathLength_MaximumSearchPathLength = 0x00003030;
            break;
    }
    // Ref0 and Ref1 will be setup below for every IME send.
    // Reference Region Height (RefHeight) - M0.5 (bits 31:24)
    // Reference Region Width (RefWidth)   - M0.5 (bits 23:16)
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 5, dw05_refWindow );
    // Maximum Search Path Length (MaxNumSU) - M1.2 (bits 15:8)
    // Max Fixed Search Path Length (LenSP)  - M1.2 (bits 7:0)
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 1, 2, dw12_MaxFixedSearchPathLength_MaximumSearchPathLength );

    // Set srcX and srcY in the message payload
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 2, as_uint(convert_ushort2(srcCoord)));
    // Set the cost precision DW
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 1, 7, MVCostPrecision | __builtin_IB_get_message_phase_dw(VMEUniversal, 1, 7));
    // Set the cost penalty table DWs.
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 2, 3, searchCostTable.x);
    __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 2, 4, searchCostTable.y);

    // Get the cost center for the ith IME/FBR from the 0th SIMD channel
    // of predMVs (the 1st predictor)
    const ushort2 CostCoord = as_ushort2(sub_group_broadcast(as_uint(convert_short2(predMV)), 0));
    // Set CostX and CostY in the message payload
    // The cost center need to be specified in QPEL precision
    // and so we need to perform PEL to QPEL conversion here
    const uint CostCenter = as_uint(CostCoord << (ushort2)2);

    //Build the VME Search Path
    GRFHandle VMESearchPath = __builtin_IB_create_message_phases(2);
    create_ime_message(VMESearchPath);

    // Do a runtime check to see if we need to issue the ith IME.
    // Get the predictor for the ith IME from the ith SIMD channel
    // of predMVs.
    GRFHandle MVMin  = __builtin_IB_create_message_phases_no_init(4);
    GRFHandle SADMin = __builtin_IB_create_message_phases_no_init(1);
    GRFHandle result = __builtin_IB_create_message_phases(RETURN_MESSAGE_NUM_GRFS);
    long srcImgInt = (long)__builtin_astype(srcImg, void*);
    long refImgInt = (long)__builtin_astype(refImg, void*);

    // i = 0
    {
        ushort2 ref0Coord = CostCoord + as_ushort2(dw01_refOffset);
        __builtin_IB_vme_send_ime(result, VMEUniversal, VMESearchPath, srcImgInt, refImgInt, as_uint(ref0Coord), dw01_refOffset, CostCenter);
        __builtin_IB_extract_mv_and_sad(MVMin, SADMin, result, __builtin_IB_vme_mb_block_type());
    }
    for (int i=1; i < MaxPredMVs; i++)
    {
        if (i < countPredMVs)
        {
            ushort2 vRefCoord = as_ushort2(sub_group_broadcast(as_uint(convert_short2(predMV)), i));
            ushort2 ref0Coord = vRefCoord + as_ushort2(dw01_refOffset);

            __builtin_IB_vme_send_ime(result, VMEUniversal, VMESearchPath, srcImgInt, refImgInt, as_uint(ref0Coord), dw01_refOffset, CostCenter);

            GRFHandle MVCurr  = __builtin_IB_create_message_phases_no_init(4);
            GRFHandle SADCurr = __builtin_IB_create_message_phases_no_init(1);
            __builtin_IB_extract_mv_and_sad(MVCurr, SADCurr, result, __builtin_IB_vme_mb_block_type());
            __builtin_IB_cmp_sads(MVCurr, SADCurr, MVMin, SADMin);
        }
    }

    // Set up the check to see if we need to do an FBR
    if( __builtin_IB_vme_subpixel_mode() != SAMPLER_VME_SUBPIXEL_MODE_INTEGER )
    {
        // Disable BME
        __builtin_IB_set_message_phase_legacy_dw(VMEUniversal, 0, 3, 0x00040000 | __builtin_IB_get_message_phase_dw(VMEUniversal, 0, 3));

        uint W00                   = __builtin_IB_get_message_phase_dw(result, 0, 0);
        uint W06                   = __builtin_IB_get_message_phase_dw(result, 0, 6);
        __private uchar* W00Bytes  = (__private uchar*)&W00;
        __private uchar* W06Bytes  = (__private uchar*)&W06;

        uint interMbMode        = W00Bytes[0];
        uint subMbShape         = W06Bytes[1];
        // No support for FBRSubPredModeVar yet, create new zeroed var
        uint subMbPredMode      = 0;
        __builtin_IB_vme_send_fbr(result, VMEUniversal, MVMin, srcImgInt, refImgInt, interMbMode, subMbShape, subMbPredMode);
    }
    else
    {
        __builtin_IB_set_message_phase_legacy(result, 1, __builtin_IB_get_message_phase(MVMin, 0));
        __builtin_IB_set_message_phase_legacy(result, 2, __builtin_IB_get_message_phase(MVMin, 1));
        __builtin_IB_set_message_phase_legacy(result, 3, __builtin_IB_get_message_phase(MVMin, 2));
        __builtin_IB_set_message_phase_legacy(result, 4, __builtin_IB_get_message_phase(MVMin, 3));

        __builtin_IB_set_message_phase_legacy(result, 5, __builtin_IB_get_message_phase(SADMin, 0));
    }

    __local uint* tmpDst = dstSearch + __builtin_IB_simd_lane_id();
    tmpDst[8]  = __builtin_IB_get_message_phase(result, 1);
    tmpDst[16] = __builtin_IB_get_message_phase(result, 2);
    tmpDst[24] = __builtin_IB_get_message_phase(result, 3);
    tmpDst[32] = __builtin_IB_get_message_phase(result, 4);
    tmpDst[40] = __builtin_IB_get_message_phase(result, 5);
}
#endif // __IGC_BUILD__

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_query_8(
    __local uint* dstSearch,
    uint countPredMVs,
    uint MVCostPrecision,
    uint2 searchCostTable,
    int2 srcCoord,
    int2 predMV,
    read_only image2d_t srcImg,
    read_only image2d_t refImg,
    sampler_t accelerator )
{
    DoMultiQuery(
        dstSearch,
        countPredMVs,
        MVCostPrecision,
        searchCostTable,
        srcCoord,
        predMV,
        srcImg,
        refImg,
        accelerator,
        8);
}

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_query_4(
    __local uint* dstSearch,
    uint countPredMVs,
    uint MVCostPrecision,
    uint2 searchCostTable,
    int2 srcCoord,
    int2 predMV,
    read_only image2d_t srcImg,
    read_only image2d_t refImg,
    sampler_t accelerator )
{
    DoMultiQuery(
        dstSearch,
        countPredMVs,
        MVCostPrecision,
        searchCostTable,
        srcCoord,
        predMV,
        srcImg,
        refImg,
        accelerator,
        4);
}

#ifdef __IGC_BUILD__
static GRFHandle create_universal_input_sic_message(bool is16x16, ushort edgeMask, int2 srcCoord)
{
    GRFHandle msg = __builtin_IB_create_message_phases(UNIVERSAL_INPUT_MESSAGE_NUM_GRFS);

    // Set the skip block type (M0.3, bit 14).
    __builtin_IB_set_message_phase_legacy_ub(msg, 0, 3*4 + 1, ((uchar)(!is16x16)) << 6);

    // Set the SAD adjustment mode - M0.3 (23:20).
    {
        uint mode = __builtin_IB_vme_sad_adjust_mode();

        if (mode == SAMPLER_VME_SAD_ADJUST_MODE_HAAR)
        {
            // Intra and Inter SAD Measure Adjustment
            uchar val = ((HAAR_TRANSFORM_ADJUSTED << 2) | HAAR_TRANSFORM_ADJUSTED) << 4;
            __builtin_IB_set_message_phase_legacy_ub(msg, 0, 3*4 + 2, val);
        }
    }

    // Set the MBIntraStruct - set the edge mask. (M1.7 - 15:8).
    {
        // the calling kernel already shifts this up 8 bits.
        __builtin_IB_set_message_phase_legacy_uw(msg, 1, 7*2 + 0, edgeMask);
    }

    // Set srcX in the message payload - M0.2 (low word).
    // Set srcY in the message payload - M0.2 (high word).
    __builtin_IB_set_message_phase_legacy_dw(msg, 0, 2, as_uint(convert_ushort2(srcCoord)));

    return msg;
}

static INLINE GRFHandle ReadPixels(read_only image2d_t intraSrcImage, int2 coord, int width, int height)
{
    // allocate a block of 64 bytes to store the read data into.
    long intraSrcImageInt = (long)__builtin_astype(intraSrcImage, void*);
    GRFHandle dst = __builtin_IB_create_message_phases((width * height) / (NUM_DWORD_IN_GRF * 4));
    __builtin_IB_media_block_rectangle_read(intraSrcImageInt, coord, width, height, dst);

    return dst;
}

static INLINE void SetSkipMVs(
    bool is16x16,
    bool isBidir,
    int  skipMV1,
    int4 skipMV4,
    int  skipMV1Bidir,
    int2 skipMV2Bidir,
    int  i,
    GRFHandle sicInput)
{
    // The 16x16 case has only one skip MV per set.
    if (is16x16)
    {
        if (isBidir)
        {
            __builtin_IB_simdMediaRegionCopy(sicInput, 0, 1, 2, skipMV1Bidir, 8*i, 2, 2, 1, 4, 2, 16);
        }
        else
        {
            int skip = intel_sub_group_shuffle(skipMV1, i);
            __builtin_IB_set_message_phase_legacy_dw(sicInput, 0, 0, skip);
        }
    }
    // 8x8 has 4 skip MVs per set.
    else
    {
        if (isBidir)
        {
            // For the bidir case, skip MVs are loaded into registers in the manner that
            // hardware expects it so, instead of shuffling, we just directly do a mov to
            // push the skip MVs into the SIC payload.
            if (i < 2)
            {
                __builtin_IB_simdMediaRegionCopy(sicInput, 0, 1, 8, skipMV2Bidir.x, 8*4*i, 8, 8, 1, 4, 8, 16);
            }
            else
            {
                __builtin_IB_simdMediaRegionCopy(sicInput, 0, 1, 8, skipMV2Bidir.y, 8*4*(i-2), 8, 8, 1, 4, 8, 16);
            }
        }
        else
        {
            int4 skip = intel_sub_group_shuffle(skipMV4, i);
            __builtin_IB_set_message_phase_legacy_dw(sicInput, 0, 0, skip.x);
            __builtin_IB_set_message_phase_legacy_dw(sicInput, 0, 2, skip.y);
            __builtin_IB_set_message_phase_legacy_dw(sicInput, 0, 4, skip.z);
            __builtin_IB_set_message_phase_legacy_dw(sicInput, 0, 6, skip.w);
        }
    }
}

//////////////////////////////////////////////////////////////////////
//
// This is the core implementation for skip checks.  8x8 and 16x16 call
// into this function with small variations.
//
// All arguments here are uniform except for skipMV and skipMVs.
//
static INLINE void DoMultiCheck(
    bool is16x16,
    bool isBidir,
    // The bottom 8 GRF are loaded with distortion values and
    // the top 3 GRF are for intra prediction modes (16x16, 8x8, 4x4).
    __local uint* dstSkipIntra, // __local uint[64 + 24];
    // How many skip MV sets are coming in.  We key off of this
    // to determine how many CRE sends to do.
    uint countSkipMVs,
    // Has the user requested intra prediction?
    uint doIntra,
    // The edge mask.  The calling kernel sets this up and will mask
    // off reading from neighbor macroblocks when we're on the edges
    // of the image.
    uint intraEdges,
    // Location of the current macroblock.
    int2 srcCoord,

    uchar bidir_weight,
    uchar skipMode,

    int  skipMV1,      // short2, used by intel_work_group_vme_mb_multi_check_16x16()
    int4 skipMV4,      // 4 short2s, used by intel_work_group_vme_mb_multi_check_8x8()
    int  skipMV1Bidir, // used by intel_work_group_vme_mb_multi_bidir_check_16x16(), laid out for hardware
    int2 skipMV2Bidir, // used by intel_work_group_vme_mb_multi_bidir_check_8x8(), laid out for hardware

    // The source and reference frames from the video.
    read_only image2d_t srcImage,
    read_only image2d_t refImage0,
    read_only image2d_t refImage1,

    // This is just an alias to srcImage.  It is set up with render surface state
    // so that we can do media block reads from it.
    read_only image2d_t intraSrcImage,

    uint MaxSkipMVs)
{
    GRFHandle universalMsg = create_universal_input_sic_message(is16x16, (ushort)intraEdges, srcCoord);

    GRFHandle sicInput     = __builtin_IB_create_message_phases(INPUT_MESSAGE_SIC_NUM_GRFS);

    GRFHandle result       = __builtin_IB_create_message_phases(RETURN_MESSAGE_NUM_GRFS);

    long srcImageInt = (long)__builtin_astype(srcImage, void*);
    long refImage0Int = (long)__builtin_astype(refImage0, void*);
    long refImage1Int = (long)__builtin_astype(refImage1, void*);

    // Set the Intra Compute Type (M1.1, bits 9:8).
    // Initially disable and turn it on if we need to do intra prediction.
    __builtin_IB_set_message_phase_legacy_ub(sicInput, 1, 1*4 + 1, INTRA_COMPUTE_TYPE_DISABLED);

    if (isBidir)
    {
        // Set the Bidirectional Weight (BiWeight) (M1.1, bits 21:16).
        __builtin_IB_set_message_phase_legacy_ub(universalMsg, 1, 1*4 + 2, bidir_weight);

        uchar val = (is16x16 ? skipMode & 0x3 : skipMode);
        // Set the skip block mask (M1.7 - 31:24).
        __builtin_IB_set_message_phase_legacy_ub(universalMsg, 1, 7*4 + 3, val);
    }
    else
    {
        uchar val = (is16x16 ? REF0_SKIP_CENTER_0 : REF0_ALL_SKIP_CENTERS);
        // Set the skip block mask (M1.7 - 31:24).
        __builtin_IB_set_message_phase_legacy_ub(universalMsg, 1, 7*4 + 3, val);
    }

    if (doIntra)
    {
        __builtin_IB_set_message_phase_legacy_ub(sicInput, 1, 1*4 + 1, INTRA_COMPUTE_TYPE_LUMA_ONLY);

        // Each block is 16x16 pixels
        // +--------+--------+--------+
        // |        |        |        |
        // |   D    |   B    |   C    |
        // |        |        |        |
        // +--------------------------+
        // |        |  Curr  |
        // |   A    |   E    |
        // |        |        |
        // +--------+--------+

        // Read pixels from the top three neighbors of the current macroblock.  Reading:
        //   Bottom right corner pixel of D
        // + All 16 bottom edge pixels of B
        // + First 8 bottom edge pixels of C
        // = 25 pixels
        // Src coord is offset -4 in the x so the read remains dword aligned
        GRFHandle topEdgePixels  = ReadPixels(intraSrcImage, srcCoord + (int2)(-4, -1), /*width*/ 32, /*height*/ 2);
        // The 16 pixels on the right edge of A (aka the left neighbor of E) are read.
        GRFHandle leftEdgePixels = ReadPixels(intraSrcImage, srcCoord + (int2)(-4, 0),  /*width*/ 4,  /*height*/ 16);

        // load left edge pixels.
        // shift three bytes over to align on the right edge of the block load then stride 4 across
        // to read each pixel down the edge.
        // mov (8) r64.0<1>:ub r10.3<32;8,4>:ub {Align1, Q1, NoMask} // #??:$193:%211
        __builtin_IB_simdMediaRegionCopy(sicInput, 64, 1, 8, leftEdgePixels, 3, 32, 8, 4, 1, 8, 64-3);
        // same as above but read the last 8 pixels down the right edge.
        // mov (8) r64.8<1>:ub r11.3<32;8,4>:ub {Align1, Q1, NoMask} // #??:$194:%212
        __builtin_IB_simdMediaRegionCopy(sicInput, 64+8, 1, 8, leftEdgePixels, 3+32, 32, 8, 4, 1, 8, 64-3-32);

        // load top edge pixels.
        // load the first 4 dwords (16 pixels) from the top edge.
        // mov (4) r63.2<1>:ud r8.1<4;4,1>:ud {Align1, Q1, NoMask} // #??:$195:%213
        __builtin_IB_simdMediaRegionCopy(sicInput, 32+8, 1, 4, topEdgePixels, 4, 4, 4, 1, 4, 4, 4);
        // load last 2 dwords (last 8 pixels).
        // mov (2) r63.6<1>:ud r8.5<2;2,1>:ud {Align1, Q1, NoMask} // #??:$196:%214
        __builtin_IB_simdMediaRegionCopy(sicInput, 32+8+16, 1, 2, topEdgePixels, 4+16, 2, 2, 1, 4, 2, 2);
        // load left corner pixel.
        // mov (1) r63.7<2>:ub r8.3<0;1,0>:ub {Align1, Q1, NoMask} // #??:$197:%215
        __builtin_IB_simdMediaRegionCopy(sicInput, 32+7, 1, 1, topEdgePixels, 3, 0, 1, 0, 1, 1, 1);
    }

    local uint *pDistortionBase = dstSkipIntra;
    local uint *pIntraBase      = &dstSkipIntra[NUM_DWORD_IN_GRF * MaxSkipMVs];

    // We will do a either 3 intra operations (16x16, 8x8, 4x4) or none.  This is why
    // the 8 iterations over the skip MV sets are split into a loop of 3 and a loop of 5.
    // The hardware is capable of doing skip and intra simultaneously.
#if (__OPENCL_C_VERSION__ >= CL_VERSION_2_0)
    __attribute__((opencl_unroll_hint(8)))
#endif
    for (int i=0; i < 3; i++)
    {
        bool doSkip = (i < countSkipMVs);

        if (doSkip | (bool)doIntra)
        {
            {
                uchar IntraLumaPartMask = 0x7; // none
                switch (i)
                {
                    case 0: IntraLumaPartMask = 0x6; break; // 16x16 intra
                    case 1: IntraLumaPartMask = 0x5; break; //   8x8 intra
                    case 2: IntraLumaPartMask = 0x3; break; //   4x4 intra
                }

                // Luma Intra Partition Mask (IntraPartMask) (M1.7 - 4:0)
                __builtin_IB_set_message_phase_legacy_ub(universalMsg, 1, 7*4 + 0, IntraLumaPartMask);
            }

            // Set the skip mode enable bit (M1.0 - 0:0).
            __builtin_IB_set_message_phase_legacy_ub(universalMsg, 1, 0*4 + 0, (uchar)doSkip);

            SetSkipMVs(
                is16x16,
                isBidir,
                skipMV1,
                skipMV4,
                skipMV1Bidir,
                skipMV2Bidir,
                i,
                sicInput);

            // combine the universal and sic inputs into one big 8 GRF payload and fire
            // off to the CRE.  Result will contain information about distortion values
            // and intra prediction modes.
            __builtin_IB_vme_send_sic(result, universalMsg, sicInput, srcImageInt, refImage0Int, refImage1Int);

            // load results into SLM

            // TODO: I'd like to emit a send (8) here instead of having to
            // shut off the top channels in simd16 since we're only writing
            // one GRF.
            if (get_sub_group_local_id() < NUM_DWORD_IN_GRF)
            {
                // Store W5.0 - 8 DWORDs (1 reg) containing distortion values.
                pDistortionBase[NUM_DWORD_IN_GRF * i + get_sub_group_local_id()] =
                    __builtin_IB_get_message_phase(result, 5);
                // Store W0.0 - 8 DWORDs (1 reg) containing intra prediction modes.
                pIntraBase[NUM_DWORD_IN_GRF * i + get_sub_group_local_id()] =
                    __builtin_IB_get_message_phase(result, 0);
            }
        }
    }

    // We've read our three intra prediction modes.  Turn off intra prediction.
    __builtin_IB_set_message_phase_legacy_ub(sicInput, 1, 1*4 + 1, INTRA_COMPUTE_TYPE_DISABLED);

#if (__OPENCL_C_VERSION__ >= CL_VERSION_2_0)
    __attribute__((opencl_unroll_hint(8)))
#endif
    for (int i=3; i < MaxSkipMVs; i++)
    {
        if (i < countSkipMVs)
        {
            SetSkipMVs(
                is16x16,
                isBidir,
                skipMV1,
                skipMV4,
                skipMV1Bidir,
                skipMV2Bidir,
                i,
                sicInput);

            __builtin_IB_vme_send_sic(result, universalMsg, sicInput, srcImageInt, refImage0Int, refImage1Int);

            // load results into SLM
            // Store W5.0 - 8 DWORDs (1 reg) containing distortion values.
            if (get_sub_group_local_id() < NUM_DWORD_IN_GRF)
            {
                pDistortionBase[NUM_DWORD_IN_GRF * i + get_sub_group_local_id()] =
                    __builtin_IB_get_message_phase(result, 5);
            }
        }
    }
}
#endif // __IGC_BUILD__

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_check_16x16(
    __local uint* dstSkipIntra, // __local uint[64 + 24];
    uint countSkipMVs,
    uint doIntra,
    uint intraEdges,
    int2 srcCoord,
    int  skipMV, // short2
    read_only image2d_t srcImage,
    read_only image2d_t refImage,
    read_only image2d_t intraSrcImage,
    sampler_t vmeAccelerator )
{
    DoMultiCheck(
        true, // is16x16
        false, // bidir
        dstSkipIntra,
        countSkipMVs,
        doIntra,
        intraEdges,
        srcCoord,
        0, // unused
        0, // unused
        skipMV,
        0,0,0, // unused
        srcImage,
        refImage,
        refImage,
        intraSrcImage,
        8);
}

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_check_8x8(
    __local uint* dstSkipIntra, // __local uint[64 + 24];
    uint countSkipMVs,
    uint doIntra,
    uint intraEdges,
    int2 srcCoord,
    int4 skipMVs, // 4 short2
    read_only image2d_t srcImage,
    read_only image2d_t refImage,
    read_only image2d_t intraSrcImage,
    sampler_t vmeAccelerator )
{
    DoMultiCheck(
        false, // is16x16
        false, // bidir
        dstSkipIntra,
        countSkipMVs,
        doIntra,
        intraEdges,
        srcCoord,
        0, 0, // unused
        0,
        skipMVs,
        0,0,
        srcImage,
        refImage,
        refImage,
        intraSrcImage,
        8);
}

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_bidir_check_16x16(
    __local uint* dstSkipIntra,
    uint countSkipMVs,
    uint doIntra,
    uint intraEdges,
    int2 srcCoord,
    uchar bidirWeight,
    uchar skipModes,
    int  skipMV,   // short2
    read_only image2d_t srcImage,
    read_only image2d_t refFwdImage,
    read_only image2d_t refBwdImage,
    read_only image2d_t intraSrcImage,
    sampler_t vmeAccelerator )
{
    DoMultiCheck(
        true, // is16x16
        true, // bidir
        dstSkipIntra,
        countSkipMVs,
        doIntra,
        intraEdges,
        srcCoord,
        bidirWeight,
        skipModes,
        0,0,
        skipMV,
        0, // unused
        srcImage,
        refFwdImage,
        refBwdImage,
        intraSrcImage,
        4);
}

INLINE void OVERLOADABLE intel_work_group_vme_mb_multi_bidir_check_8x8(
    __local uint* dstSkipIntra,
    uint countSkipMVs,
    uint doIntra,
    uint intraEdges,
    int2 srcCoord,
    uchar bidirWeight,
    uchar skipModes,
    int2 skipMVs,    // 4 short2
    read_only image2d_t srcImage,
    read_only image2d_t refFwdImage,
    read_only image2d_t refBwdImage,
    read_only image2d_t intraSrcImage,
    sampler_t vmeAccelerator )
{
    DoMultiCheck(
        false, // is16x16
        true, // bidir
        dstSkipIntra,
        countSkipMVs,
        doIntra,
        intraEdges,
        srcCoord,
        bidirWeight,
        skipModes,
        0,0,0,
        skipMVs,
        srcImage,
        refFwdImage,
        refBwdImage,
        intraSrcImage,
        4);
}
