vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity
Files
9695e81f9c815e5ab02cfe03ecd21c41b635cea7
RedBear-OS/local/recipes/dev/spirv-llvm-translator/source/lib/SPIRV/OCLUtil.cpp
T

1625 lines
65 KiB
C++
Raw Blame History

//===- OCLUtil.cpp - OCL Utilities ----------------------------------------===//
//
// The LLVM/SPIRV Translator
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
// Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimers in the documentation
// and/or other materials provided with the distribution.
// Neither the names of Advanced Micro Devices, Inc., nor the names of its
// contributors may be used to endorse or promote products derived from this
// Software without specific prior written permission.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
// THE SOFTWARE.
//
//===----------------------------------------------------------------------===//
//
// This file implements OCL utility functions.
//
//===----------------------------------------------------------------------===//
#include "OCLUtil.h"
#include "SPIRVEntry.h"
#include "SPIRVFunction.h"
#include "SPIRVInstruction.h"
#include "SPIRVInternal.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/Instructions.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#define DEBUG_TYPE "oclutil"
using namespace llvm;
using namespace SPIRV;
namespace OCLUtil {
#ifndef SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#define SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE SPIRAS_Private
#endif
#ifndef SPIRV_QUEUE_T_ADDR_SPACE
#define SPIRV_QUEUE_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
#ifndef SPIRV_EVENT_T_ADDR_SPACE
#define SPIRV_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
#ifndef SPIRV_AVC_INTEL_T_ADDR_SPACE
#define SPIRV_AVC_INTEL_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
#ifndef SPIRV_CLK_EVENT_T_ADDR_SPACE
#define SPIRV_CLK_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
#ifndef SPIRV_SAMPLER_T_ADDR_SPACE
#define SPIRV_SAMPLER_T_ADDR_SPACE SPIRAS_Constant
#endif
#ifndef SPIRV_RESERVE_ID_T_ADDR_SPACE
#define SPIRV_RESERVE_ID_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
// Excerpt from SPIR 2.0 spec.:
// Pipe objects are represented using pointers to the opaque %opencl.pipe LLVM
// structure type which reside in the global address space.
#ifndef SPIRV_PIPE_ADDR_SPACE
#define SPIRV_PIPE_ADDR_SPACE SPIRAS_Global
#endif
// Excerpt from SPIR 2.0 spec.:
// Note: Images data types reside in global memory and hence should be marked
// as such in the "kernel arg addr space" metadata.
#ifndef SPIRV_IMAGE_ADDR_SPACE
#define SPIRV_IMAGE_ADDR_SPACE SPIRAS_Global
#endif
} // namespace OCLUtil
///////////////////////////////////////////////////////////////////////////////
//
// Map definitions
//
///////////////////////////////////////////////////////////////////////////////
using namespace OCLUtil;
namespace SPIRV {
template <> void SPIRVMap<OCLMemFenceKind, MemorySemanticsMask>::init() {
add(OCLMF_Local, MemorySemanticsWorkgroupMemoryMask);
add(OCLMF_Global, MemorySemanticsCrossWorkgroupMemoryMask);
add(OCLMF_Image, MemorySemanticsImageMemoryMask);
}
template <>
void SPIRVMap<OCLMemFenceExtendedKind, MemorySemanticsMask>::init() {
add(OCLMFEx_Local, MemorySemanticsWorkgroupMemoryMask);
add(OCLMFEx_Global, MemorySemanticsCrossWorkgroupMemoryMask);
add(OCLMFEx_Local_Global, MemorySemanticsWorkgroupMemoryMask |
MemorySemanticsCrossWorkgroupMemoryMask);
add(OCLMFEx_Image, MemorySemanticsImageMemoryMask);
add(OCLMFEx_Image_Local,
MemorySemanticsWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
add(OCLMFEx_Image_Global,
MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
add(OCLMFEx_Image_Local_Global, MemorySemanticsWorkgroupMemoryMask |
MemorySemanticsCrossWorkgroupMemoryMask |
MemorySemanticsImageMemoryMask);
}
template <>
void SPIRVMap<OCLMemOrderKind, unsigned, MemorySemanticsMask>::init() {
add(OCLMO_relaxed, MemorySemanticsMaskNone);
add(OCLMO_acquire, MemorySemanticsAcquireMask);
add(OCLMO_release, MemorySemanticsReleaseMask);
add(OCLMO_acq_rel, MemorySemanticsAcquireReleaseMask);
add(OCLMO_seq_cst, MemorySemanticsSequentiallyConsistentMask);
}
template <> void SPIRVMap<OCLScopeKind, Scope>::init() {
add(OCLMS_work_item, ScopeInvocation);
add(OCLMS_work_group, ScopeWorkgroup);
add(OCLMS_device, ScopeDevice);
add(OCLMS_all_svm_devices, ScopeCrossDevice);
add(OCLMS_sub_group, ScopeSubgroup);
}
template <> void SPIRVMap<std::string, Scope>::init() {
add("work_item", ScopeInvocation);
add("workgroup", ScopeWorkgroup);
add("device", ScopeDevice);
add("all_svm_devices", ScopeCrossDevice);
add("subgroup", ScopeSubgroup);
}
template <> void SPIRVMap<std::string, SPIRVGroupOperationKind>::init() {
add("reduce", GroupOperationReduce);
add("scan_inclusive", GroupOperationInclusiveScan);
add("scan_exclusive", GroupOperationExclusiveScan);
add("ballot_bit_count", GroupOperationReduce);
add("ballot_inclusive_scan", GroupOperationInclusiveScan);
add("ballot_exclusive_scan", GroupOperationExclusiveScan);
add("non_uniform_reduce", GroupOperationReduce);
add("non_uniform_scan_inclusive", GroupOperationInclusiveScan);
add("non_uniform_scan_exclusive", GroupOperationExclusiveScan);
add("non_uniform_reduce_logical", GroupOperationReduce);
add("non_uniform_scan_inclusive_logical", GroupOperationInclusiveScan);
add("non_uniform_scan_exclusive_logical", GroupOperationExclusiveScan);
add("clustered_reduce", GroupOperationClusteredReduce);
}
template <> void SPIRVMap<std::string, SPIRVFPRoundingModeKind>::init() {
add("rte", FPRoundingModeRTE);
add("rtz", FPRoundingModeRTZ);
add("rtp", FPRoundingModeRTP);
add("rtn", FPRoundingModeRTN);
}
template <> void SPIRVMap<OclExt::Kind, std::string>::init() {
#define _SPIRV_OP(x) add(OclExt::x, #x);
_SPIRV_OP(cl_images)
_SPIRV_OP(cl_doubles)
_SPIRV_OP(cl_khr_int64_base_atomics)
_SPIRV_OP(cl_khr_int64_extended_atomics)
_SPIRV_OP(cl_khr_fp16)
_SPIRV_OP(cl_khr_gl_sharing)
_SPIRV_OP(cl_khr_gl_event)
_SPIRV_OP(cl_khr_d3d10_sharing)
_SPIRV_OP(cl_khr_media_sharing)
_SPIRV_OP(cl_khr_d3d11_sharing)
_SPIRV_OP(cl_khr_global_int32_base_atomics)
_SPIRV_OP(cl_khr_global_int32_extended_atomics)
_SPIRV_OP(cl_khr_local_int32_base_atomics)
_SPIRV_OP(cl_khr_local_int32_extended_atomics)
_SPIRV_OP(cl_khr_byte_addressable_store)
_SPIRV_OP(cl_khr_3d_image_writes)
_SPIRV_OP(cl_khr_gl_msaa_sharing)
_SPIRV_OP(cl_khr_depth_images)
_SPIRV_OP(cl_khr_gl_depth_images)
_SPIRV_OP(cl_khr_subgroups)
_SPIRV_OP(cl_khr_mipmap_image)
_SPIRV_OP(cl_khr_mipmap_image_writes)
_SPIRV_OP(cl_khr_egl_event)
_SPIRV_OP(cl_khr_srgb_image_writes)
_SPIRV_OP(cl_khr_extended_bit_ops)
#undef _SPIRV_OP
}
template <> void SPIRVMap<OclExt::Kind, SPIRVCapabilityKind>::init() {
add(OclExt::cl_images, CapabilityImageBasic);
add(OclExt::cl_doubles, CapabilityFloat64);
add(OclExt::cl_khr_int64_base_atomics, CapabilityInt64Atomics);
add(OclExt::cl_khr_int64_extended_atomics, CapabilityInt64Atomics);
add(OclExt::cl_khr_fp16, CapabilityFloat16);
add(OclExt::cl_khr_subgroups, CapabilityGroups);
add(OclExt::cl_khr_mipmap_image, CapabilityImageMipmap);
add(OclExt::cl_khr_mipmap_image_writes, CapabilityImageMipmap);
add(OclExt::cl_khr_extended_bit_ops, CapabilityBitInstructions);
}
/// Map OpenCL work functions to SPIR-V builtin variables.
template <> void SPIRVMap<std::string, SPIRVBuiltinVariableKind>::init() {
add("get_work_dim", BuiltInWorkDim);
add("get_global_size", BuiltInGlobalSize);
add("get_global_id", BuiltInGlobalInvocationId);
add("get_global_offset", BuiltInGlobalOffset);
add("get_local_size", BuiltInWorkgroupSize);
add("get_enqueued_local_size", BuiltInEnqueuedWorkgroupSize);
add("get_local_id", BuiltInLocalInvocationId);
add("get_num_groups", BuiltInNumWorkgroups);
add("get_group_id", BuiltInWorkgroupId);
add("get_global_linear_id", BuiltInGlobalLinearId);
add("get_local_linear_id", BuiltInLocalInvocationIndex);
// cl_khr_subgroups
add("get_sub_group_size", BuiltInSubgroupSize);
add("get_max_sub_group_size", BuiltInSubgroupMaxSize);
add("get_num_sub_groups", BuiltInNumSubgroups);
add("get_enqueued_num_sub_groups", BuiltInNumEnqueuedSubgroups);
add("get_sub_group_id", BuiltInSubgroupId);
add("get_sub_group_local_id", BuiltInSubgroupLocalInvocationId);
// cl_khr_subgroup_ballot
add("get_sub_group_eq_mask", BuiltInSubgroupEqMask);
add("get_sub_group_ge_mask", BuiltInSubgroupGeMask);
add("get_sub_group_gt_mask", BuiltInSubgroupGtMask);
add("get_sub_group_le_mask", BuiltInSubgroupLeMask);
add("get_sub_group_lt_mask", BuiltInSubgroupLtMask);
}
// Maps uniqued OCL builtin function name to SPIR-V op code.
// A uniqued OCL builtin function name may be different from the real
// OCL builtin function name. e.g. instead of atomic_min, atomic_umin
// is used for atomic_min with unsigned integer parameter.
// work_group_ and sub_group_ functions are unified as group_ functions
// except work_group_barrier.
class SPIRVInstruction;
template <> void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
#define _SPIRV_OP(x, y) add("atom_" #x, OpAtomic##y);
// cl_khr_int64_base_atomics builtins
_SPIRV_OP(add, IAdd)
_SPIRV_OP(sub, ISub)
_SPIRV_OP(xchg, Exchange)
_SPIRV_OP(dec, IDecrement)
_SPIRV_OP(inc, IIncrement)
_SPIRV_OP(cmpxchg, CompareExchange)
// cl_khr_int64_extended_atomics builtins
_SPIRV_OP(min, SMin)
_SPIRV_OP(max, SMax)
_SPIRV_OP(and, And)
_SPIRV_OP(or, Or)
_SPIRV_OP(xor, Xor)
#undef _SPIRV_OP
#define _SPIRV_OP(x, y) add("atomic_" #x, Op##y);
// CL 2.0 atomic builtins
_SPIRV_OP(flag_test_and_set_explicit, AtomicFlagTestAndSet)
_SPIRV_OP(flag_clear_explicit, AtomicFlagClear)
_SPIRV_OP(load_explicit, AtomicLoad)
_SPIRV_OP(store_explicit, AtomicStore)
_SPIRV_OP(exchange_explicit, AtomicExchange)
_SPIRV_OP(compare_exchange_strong_explicit, AtomicCompareExchange)
_SPIRV_OP(compare_exchange_weak_explicit, AtomicCompareExchangeWeak)
_SPIRV_OP(inc, AtomicIIncrement)
_SPIRV_OP(dec, AtomicIDecrement)
_SPIRV_OP(fetch_add_explicit, AtomicIAdd)
_SPIRV_OP(fetch_sub_explicit, AtomicISub)
_SPIRV_OP(fetch_umin_explicit, AtomicUMin)
_SPIRV_OP(fetch_umax_explicit, AtomicUMax)
_SPIRV_OP(fetch_min_explicit, AtomicSMin)
_SPIRV_OP(fetch_max_explicit, AtomicSMax)
_SPIRV_OP(fetch_and_explicit, AtomicAnd)
_SPIRV_OP(fetch_or_explicit, AtomicOr)
_SPIRV_OP(fetch_xor_explicit, AtomicXor)
#undef _SPIRV_OP
#define _SPIRV_OP(x, y) add(#x, Op##y);
_SPIRV_OP(dot, Dot)
_SPIRV_OP(async_work_group_copy, GroupAsyncCopy)
_SPIRV_OP(async_work_group_strided_copy, GroupAsyncCopy)
_SPIRV_OP(wait_group_events, GroupWaitEvents)
_SPIRV_OP(isequal, FOrdEqual)
_SPIRV_OP(isnotequal, FUnordNotEqual)
_SPIRV_OP(isgreater, FOrdGreaterThan)
_SPIRV_OP(isgreaterequal, FOrdGreaterThanEqual)
_SPIRV_OP(isless, FOrdLessThan)
_SPIRV_OP(islessequal, FOrdLessThanEqual)
_SPIRV_OP(islessgreater, FOrdNotEqual)
_SPIRV_OP(isordered, Ordered)
_SPIRV_OP(isunordered, Unordered)
_SPIRV_OP(isfinite, IsFinite)
_SPIRV_OP(isinf, IsInf)
_SPIRV_OP(isnan, IsNan)
_SPIRV_OP(isnormal, IsNormal)
_SPIRV_OP(signbit, SignBitSet)
_SPIRV_OP(any, Any)
_SPIRV_OP(all, All)
_SPIRV_OP(popcount, BitCount)
_SPIRV_OP(get_fence, GenericPtrMemSemantics)
// CL 2.0 kernel enqueue builtins
_SPIRV_OP(enqueue_marker, EnqueueMarker)
_SPIRV_OP(enqueue_kernel, EnqueueKernel)
_SPIRV_OP(get_kernel_sub_group_count_for_ndrange_impl,
GetKernelNDrangeSubGroupCount)
_SPIRV_OP(get_kernel_max_sub_group_size_for_ndrange_impl,
GetKernelNDrangeMaxSubGroupSize)
_SPIRV_OP(get_kernel_work_group_size_impl, GetKernelWorkGroupSize)
_SPIRV_OP(get_kernel_preferred_work_group_size_multiple_impl,
GetKernelPreferredWorkGroupSizeMultiple)
_SPIRV_OP(retain_event, RetainEvent)
_SPIRV_OP(release_event, ReleaseEvent)
_SPIRV_OP(create_user_event, CreateUserEvent)
_SPIRV_OP(is_valid_event, IsValidEvent)
_SPIRV_OP(set_user_event_status, SetUserEventStatus)
_SPIRV_OP(capture_event_profiling_info, CaptureEventProfilingInfo)
_SPIRV_OP(get_default_queue, GetDefaultQueue)
_SPIRV_OP(ndrange_1D, BuildNDRange)
_SPIRV_OP(ndrange_2D, BuildNDRange)
_SPIRV_OP(ndrange_3D, BuildNDRange)
// Generic Address Space Casts
_SPIRV_OP(to_global, GenericCastToPtrExplicit)
_SPIRV_OP(to_local, GenericCastToPtrExplicit)
_SPIRV_OP(to_private, GenericCastToPtrExplicit)
// CL 2.0 pipe builtins
_SPIRV_OP(read_pipe_2, ReadPipe)
_SPIRV_OP(write_pipe_2, WritePipe)
_SPIRV_OP(read_pipe_2_bl, ReadPipeBlockingINTEL)
_SPIRV_OP(write_pipe_2_bl, WritePipeBlockingINTEL)
_SPIRV_OP(read_pipe_4, ReservedReadPipe)
_SPIRV_OP(write_pipe_4, ReservedWritePipe)
_SPIRV_OP(reserve_read_pipe, ReserveReadPipePackets)
_SPIRV_OP(reserve_write_pipe, ReserveWritePipePackets)
_SPIRV_OP(commit_read_pipe, CommitReadPipe)
_SPIRV_OP(commit_write_pipe, CommitWritePipe)
_SPIRV_OP(is_valid_reserve_id, IsValidReserveId)
_SPIRV_OP(group_reserve_read_pipe, GroupReserveReadPipePackets)
_SPIRV_OP(group_reserve_write_pipe, GroupReserveWritePipePackets)
_SPIRV_OP(group_commit_read_pipe, GroupCommitReadPipe)
_SPIRV_OP(group_commit_write_pipe, GroupCommitWritePipe)
_SPIRV_OP(get_pipe_num_packets_ro, GetNumPipePackets)
_SPIRV_OP(get_pipe_num_packets_wo, GetNumPipePackets)
_SPIRV_OP(get_pipe_max_packets_ro, GetMaxPipePackets)
_SPIRV_OP(get_pipe_max_packets_wo, GetMaxPipePackets)
// CL 2.0 workgroup builtins
_SPIRV_OP(group_all, GroupAll)
_SPIRV_OP(group_any, GroupAny)
_SPIRV_OP(group_broadcast, GroupBroadcast)
_SPIRV_OP(group_iadd, GroupIAdd)
_SPIRV_OP(group_fadd, GroupFAdd)
_SPIRV_OP(group_fmin, GroupFMin)
_SPIRV_OP(group_umin, GroupUMin)
_SPIRV_OP(group_smin, GroupSMin)
_SPIRV_OP(group_fmax, GroupFMax)
_SPIRV_OP(group_umax, GroupUMax)
_SPIRV_OP(group_smax, GroupSMax)
_SPIRV_OP(group_imul, GroupIMulKHR)
_SPIRV_OP(group_fmul, GroupFMulKHR)
_SPIRV_OP(group_ibitwise_and, GroupBitwiseAndKHR)
_SPIRV_OP(group_ibitwise_or, GroupBitwiseOrKHR)
_SPIRV_OP(group_ibitwise_xor, GroupBitwiseXorKHR)
_SPIRV_OP(group_ilogical_and, GroupLogicalAndKHR)
_SPIRV_OP(group_ilogical_or, GroupLogicalOrKHR)
_SPIRV_OP(group_ilogical_xor, GroupLogicalXorKHR)
// CL image builtins
_SPIRV_OP(SampledImage, SampledImage)
_SPIRV_OP(ImageSampleExplicitLod, ImageSampleExplicitLod)
_SPIRV_OP(read_image, ImageRead)
_SPIRV_OP(write_image, ImageWrite)
_SPIRV_OP(get_image_channel_data_type, ImageQueryFormat)
_SPIRV_OP(get_image_channel_order, ImageQueryOrder)
_SPIRV_OP(get_image_num_mip_levels, ImageQueryLevels)
_SPIRV_OP(get_image_num_samples, ImageQuerySamples)
// Intel Subgroups builtins
_SPIRV_OP(intel_sub_group_shuffle, SubgroupShuffleINTEL)
_SPIRV_OP(intel_sub_group_shuffle_down, SubgroupShuffleDownINTEL)
_SPIRV_OP(intel_sub_group_shuffle_up, SubgroupShuffleUpINTEL)
_SPIRV_OP(intel_sub_group_shuffle_xor, SubgroupShuffleXorINTEL)
// Intel media_block_io builtins
_SPIRV_OP(intel_sub_group_media_block_read, SubgroupImageMediaBlockReadINTEL)
_SPIRV_OP(intel_sub_group_media_block_write,
SubgroupImageMediaBlockWriteINTEL)
// cl_khr_subgroup_non_uniform_vote
_SPIRV_OP(group_elect, GroupNonUniformElect)
_SPIRV_OP(group_non_uniform_all, GroupNonUniformAll)
_SPIRV_OP(group_non_uniform_any, GroupNonUniformAny)
_SPIRV_OP(group_non_uniform_all_equal, GroupNonUniformAllEqual)
// cl_khr_subgroup_ballot
_SPIRV_OP(group_non_uniform_broadcast, GroupNonUniformBroadcast)
_SPIRV_OP(group_broadcast_first, GroupNonUniformBroadcastFirst)
_SPIRV_OP(group_ballot, GroupNonUniformBallot)
_SPIRV_OP(group_inverse_ballot, GroupNonUniformInverseBallot)
_SPIRV_OP(group_ballot_bit_extract, GroupNonUniformBallotBitExtract)
_SPIRV_OP(group_ballot_bit_count_iadd, GroupNonUniformBallotBitCount)
_SPIRV_OP(group_ballot_find_lsb, GroupNonUniformBallotFindLSB)
_SPIRV_OP(group_ballot_find_msb, GroupNonUniformBallotFindMSB)
// cl_khr_subgroup_non_uniform_arithmetic
_SPIRV_OP(group_non_uniform_iadd, GroupNonUniformIAdd)
_SPIRV_OP(group_non_uniform_fadd, GroupNonUniformFAdd)
_SPIRV_OP(group_non_uniform_imul, GroupNonUniformIMul)
_SPIRV_OP(group_non_uniform_fmul, GroupNonUniformFMul)
_SPIRV_OP(group_non_uniform_smin, GroupNonUniformSMin)
_SPIRV_OP(group_non_uniform_umin, GroupNonUniformUMin)
_SPIRV_OP(group_non_uniform_fmin, GroupNonUniformFMin)
_SPIRV_OP(group_non_uniform_smax, GroupNonUniformSMax)
_SPIRV_OP(group_non_uniform_umax, GroupNonUniformUMax)
_SPIRV_OP(group_non_uniform_fmax, GroupNonUniformFMax)
_SPIRV_OP(group_non_uniform_iand, GroupNonUniformBitwiseAnd)
_SPIRV_OP(group_non_uniform_ior, GroupNonUniformBitwiseOr)
_SPIRV_OP(group_non_uniform_ixor, GroupNonUniformBitwiseXor)
_SPIRV_OP(group_non_uniform_logical_iand, GroupNonUniformLogicalAnd)
_SPIRV_OP(group_non_uniform_logical_ior, GroupNonUniformLogicalOr)
_SPIRV_OP(group_non_uniform_logical_ixor, GroupNonUniformLogicalXor)
// cl_khr_subgroup_shuffle
_SPIRV_OP(group_shuffle, GroupNonUniformShuffle)
_SPIRV_OP(group_shuffle_xor, GroupNonUniformShuffleXor)
// cl_khr_subgroup_shuffle_relative
_SPIRV_OP(group_shuffle_up, GroupNonUniformShuffleUp)
_SPIRV_OP(group_shuffle_down, GroupNonUniformShuffleDown)
// cl_khr_subgroup_rotate
_SPIRV_OP(group_rotate, GroupNonUniformRotateKHR)
_SPIRV_OP(group_clustered_rotate, GroupNonUniformRotateKHR)
// cl_khr_extended_bit_ops
_SPIRV_OP(bitfield_insert, BitFieldInsert)
_SPIRV_OP(bitfield_extract_signed, BitFieldSExtract)
_SPIRV_OP(bitfield_extract_unsigned, BitFieldUExtract)
_SPIRV_OP(bit_reverse, BitReverse)
// cl_khr_split_work_group_barrier
_SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL)
_SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL)
#undef _SPIRV_OP
}
template <> void SPIRVMap<std::string, Op, OCL12Builtin>::init() {
#define _SPIRV_OP(x, y) add(#x, Op##y);
_SPIRV_OP(add, AtomicIAdd)
_SPIRV_OP(sub, AtomicISub)
_SPIRV_OP(xchg, AtomicExchange)
_SPIRV_OP(cmpxchg, AtomicCompareExchange)
_SPIRV_OP(inc, AtomicIIncrement)
_SPIRV_OP(dec, AtomicIDecrement)
_SPIRV_OP(min, AtomicSMin)
_SPIRV_OP(max, AtomicSMax)
_SPIRV_OP(umin, AtomicUMin)
_SPIRV_OP(umax, AtomicUMax)
_SPIRV_OP(and, AtomicAnd)
_SPIRV_OP(or, AtomicOr)
_SPIRV_OP(xor, AtomicXor)
#undef _SPIRV_OP
}
// SPV_INTEL_device_side_avc_motion_estimation extension builtins
class SPIRVSubgroupsAVCIntelInst;
template <> void SPIRVMap<std::string, Op, SPIRVSubgroupsAVCIntelInst>::init() {
// Here is a workaround for a bug in the specification:
// 'avc' missed in 'intel_sub_group_avc' prefix.
add("intel_sub_group_ime_ref_window_size",
OpSubgroupAvcImeRefWindowSizeINTEL);
#define _SPIRV_OP(x, y) add("intel_sub_group_avc_" #x, OpSubgroupAvc##y##INTEL);
// Initialization phase functions
_SPIRV_OP(ime_initialize, ImeInitialize)
_SPIRV_OP(fme_initialize, FmeInitialize)
_SPIRV_OP(bme_initialize, BmeInitialize)
_SPIRV_OP(sic_initialize, SicInitialize)
// Result and payload types conversion functions
_SPIRV_OP(mce_convert_to_ime_payload, MceConvertToImePayload)
_SPIRV_OP(mce_convert_to_ime_result, MceConvertToImeResult)
_SPIRV_OP(mce_convert_to_ref_payload, MceConvertToRefPayload)
_SPIRV_OP(mce_convert_to_ref_result, MceConvertToRefResult)
_SPIRV_OP(mce_convert_to_sic_payload, MceConvertToSicPayload)
_SPIRV_OP(mce_convert_to_sic_result, MceConvertToSicResult)
_SPIRV_OP(ime_convert_to_mce_payload, ImeConvertToMcePayload)
_SPIRV_OP(ime_convert_to_mce_result, ImeConvertToMceResult)
_SPIRV_OP(ref_convert_to_mce_payload, RefConvertToMcePayload)
_SPIRV_OP(ref_convert_to_mce_result, RefConvertToMceResult)
_SPIRV_OP(sic_convert_to_mce_payload, SicConvertToMcePayload)
_SPIRV_OP(sic_convert_to_mce_result, SicConvertToMceResult)
#undef _SPIRV_OP
// MCE instructions
#define _SPIRV_OP(x, y) \
add("intel_sub_group_avc_mce_" #x, OpSubgroupAvcMce##y##INTEL);
_SPIRV_OP(get_default_inter_base_multi_reference_penalty,
GetDefaultInterBaseMultiReferencePenalty)
_SPIRV_OP(set_inter_base_multi_reference_penalty,
SetInterBaseMultiReferencePenalty)
_SPIRV_OP(get_default_inter_shape_penalty, GetDefaultInterShapePenalty)
_SPIRV_OP(set_inter_shape_penalty, SetInterShapePenalty)
_SPIRV_OP(get_default_inter_direction_penalty,
GetDefaultInterDirectionPenalty)
_SPIRV_OP(set_inter_direction_penalty, SetInterDirectionPenalty)
_SPIRV_OP(get_default_intra_luma_shape_penalty,
GetDefaultIntraLumaShapePenalty)
_SPIRV_OP(get_default_inter_motion_vector_cost_table,
GetDefaultInterMotionVectorCostTable)
_SPIRV_OP(get_default_high_penalty_cost_table, GetDefaultHighPenaltyCostTable)
_SPIRV_OP(get_default_medium_penalty_cost_table,
GetDefaultMediumPenaltyCostTable)
_SPIRV_OP(get_default_low_penalty_cost_table, GetDefaultLowPenaltyCostTable)
_SPIRV_OP(set_motion_vector_cost_function, SetMotionVectorCostFunction)
_SPIRV_OP(get_default_intra_luma_mode_penalty, GetDefaultIntraLumaModePenalty)
_SPIRV_OP(get_default_non_dc_luma_intra_penalty,
GetDefaultNonDcLumaIntraPenalty)
_SPIRV_OP(get_default_intra_chroma_mode_base_penalty,
GetDefaultIntraChromaModeBasePenalty)
_SPIRV_OP(set_ac_only_haar, SetAcOnlyHaar)
_SPIRV_OP(set_source_interlaced_field_polarity,
SetSourceInterlacedFieldPolarity)
_SPIRV_OP(set_single_reference_interlaced_field_polarity,
SetSingleReferenceInterlacedFieldPolarity)
_SPIRV_OP(set_dual_reference_interlaced_field_polarities,
SetDualReferenceInterlacedFieldPolarities)
_SPIRV_OP(get_motion_vectors, GetMotionVectors)
_SPIRV_OP(get_inter_distortions, GetInterDistortions)
_SPIRV_OP(get_best_inter_distortion, GetBestInterDistortions)
_SPIRV_OP(get_inter_major_shape, GetInterMajorShape)
_SPIRV_OP(get_inter_minor_shapes, GetInterMinorShape)
_SPIRV_OP(get_inter_directions, GetInterDirections)
_SPIRV_OP(get_inter_motion_vector_count, GetInterMotionVectorCount)
_SPIRV_OP(get_inter_reference_ids, GetInterReferenceIds)
_SPIRV_OP(get_inter_reference_interlaced_field_polarities,
GetInterReferenceInterlacedFieldPolarities)
#undef _SPIRV_OP
// IME instructions
#define _SPIRV_OP(x, y) \
add("intel_sub_group_avc_ime_" #x, OpSubgroupAvcIme##y##INTEL);
_SPIRV_OP(set_single_reference, SetSingleReference)
_SPIRV_OP(set_dual_reference, SetDualReference)
_SPIRV_OP(ref_window_size, RefWindowSize)
_SPIRV_OP(adjust_ref_offset, AdjustRefOffset)
_SPIRV_OP(set_max_motion_vector_count, SetMaxMotionVectorCount)
_SPIRV_OP(set_unidirectional_mix_disable, SetUnidirectionalMixDisable)
_SPIRV_OP(set_early_search_termination_threshold,
SetEarlySearchTerminationThreshold)
_SPIRV_OP(set_weighted_sad, SetWeightedSad)
_SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
_SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
_SPIRV_OP(evaluate_with_single_reference_streamin,
EvaluateWithSingleReferenceStreamin)
_SPIRV_OP(evaluate_with_dual_reference_streamin,
EvaluateWithDualReferenceStreamin)
_SPIRV_OP(evaluate_with_single_reference_streamout,
EvaluateWithSingleReferenceStreamout)
_SPIRV_OP(evaluate_with_dual_reference_streamout,
EvaluateWithDualReferenceStreamout)
_SPIRV_OP(evaluate_with_single_reference_streaminout,
EvaluateWithSingleReferenceStreaminout)
_SPIRV_OP(evaluate_with_dual_reference_streaminout,
EvaluateWithDualReferenceStreaminout)
_SPIRV_OP(get_single_reference_streamin, GetSingleReferenceStreamin)
_SPIRV_OP(get_dual_reference_streamin, GetDualReferenceStreamin)
_SPIRV_OP(strip_single_reference_streamout, StripSingleReferenceStreamout)
_SPIRV_OP(strip_dual_reference_streamout, StripDualReferenceStreamout)
_SPIRV_OP(get_border_reached, GetBorderReached)
_SPIRV_OP(get_truncated_search_indication, GetTruncatedSearchIndication)
_SPIRV_OP(get_unidirectional_early_search_termination,
GetUnidirectionalEarlySearchTermination)
_SPIRV_OP(get_weighting_pattern_minimum_motion_vector,
GetWeightingPatternMinimumMotionVector)
_SPIRV_OP(get_weighting_pattern_minimum_distortion,
GetWeightingPatternMinimumDistortion)
#undef _SPIRV_OP
#define _SPIRV_OP(x, y) \
add("intel_sub_group_avc_ime_get_streamout_major_shape_" #x, \
OpSubgroupAvcImeGetStreamout##y##INTEL);
_SPIRV_OP(motion_vectors_single_reference,
SingleReferenceMajorShapeMotionVectors)
_SPIRV_OP(distortions_single_reference, SingleReferenceMajorShapeDistortions)
_SPIRV_OP(reference_ids_single_reference,
SingleReferenceMajorShapeReferenceIds)
_SPIRV_OP(motion_vectors_dual_reference, DualReferenceMajorShapeMotionVectors)
_SPIRV_OP(distortions_dual_reference, DualReferenceMajorShapeDistortions)
_SPIRV_OP(reference_ids_dual_reference, DualReferenceMajorShapeReferenceIds)
#undef _SPIRV_OP
// REF instructions
#define _SPIRV_OP(x, y) \
add("intel_sub_group_avc_ref_" #x, OpSubgroupAvcRef##y##INTEL);
_SPIRV_OP(set_bidirectional_mix_disable, SetBidirectionalMixDisable)
_SPIRV_OP(set_bilinear_filter_enable, SetBilinearFilterEnable)
_SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
_SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
_SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference)
_SPIRV_OP(evaluate_with_multi_reference_interlaced,
EvaluateWithMultiReferenceInterlaced)
#undef _SPIRV_OP
// SIC instructions
#define _SPIRV_OP(x, y) \
add("intel_sub_group_avc_sic_" #x, OpSubgroupAvcSic##y##INTEL);
_SPIRV_OP(configure_skc, ConfigureSkc)
_SPIRV_OP(configure_ipe_luma, ConfigureIpeLuma)
_SPIRV_OP(configure_ipe_luma_chroma, ConfigureIpeLumaChroma)
_SPIRV_OP(get_motion_vector_mask, GetMotionVectorMask)
_SPIRV_OP(set_intra_luma_shape_penalty, SetIntraLumaShapePenalty)
_SPIRV_OP(set_intra_luma_mode_cost_function, SetIntraLumaModeCostFunction)
_SPIRV_OP(set_intra_chroma_mode_cost_function, SetIntraChromaModeCostFunction)
_SPIRV_OP(set_skc_bilinear_filter_enable, SetBilinearFilterEnable)
_SPIRV_OP(set_skc_forward_transform_enable, SetSkcForwardTransformEnable)
_SPIRV_OP(set_block_based_raw_skip_sad, SetBlockBasedRawSkipSad)
_SPIRV_OP(evaluate_ipe, EvaluateIpe)
_SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
_SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
_SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference)
_SPIRV_OP(evaluate_with_multi_reference_interlaced,
EvaluateWithMultiReferenceInterlaced)
_SPIRV_OP(get_ipe_luma_shape, GetIpeLumaShape)
_SPIRV_OP(get_best_ipe_luma_distortion, GetBestIpeLumaDistortion)
_SPIRV_OP(get_best_ipe_chroma_distortion, GetBestIpeChromaDistortion)
_SPIRV_OP(get_packed_ipe_luma_modes, GetPackedIpeLumaModes)
_SPIRV_OP(get_ipe_chroma_mode, GetIpeChromaMode)
_SPIRV_OP(get_packed_skc_luma_count_threshold, GetPackedSkcLumaCountThreshold)
_SPIRV_OP(get_packed_skc_luma_sum_threshold, GetPackedSkcLumaSumThreshold)
_SPIRV_OP(get_inter_raw_sads, GetInterRawSads)
#undef _SPIRV_OP
}
template <> void SPIRVMap<std::string, Op, OCLOpaqueType>::init() {
add("opencl.event_t", OpTypeEvent);
add("opencl.pipe_t", OpTypePipe);
add("opencl.clk_event_t", OpTypeDeviceEvent);
add("opencl.reserve_id_t", OpTypeReserveId);
add("opencl.queue_t", OpTypeQueue);
add("opencl.sampler_t", OpTypeSampler);
}
template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() {
add(llvm::AtomicRMWInst::Xchg, OpAtomicExchange);
add(llvm::AtomicRMWInst::Add, OpAtomicIAdd);
add(llvm::AtomicRMWInst::Sub, OpAtomicISub);
add(llvm::AtomicRMWInst::And, OpAtomicAnd);
add(llvm::AtomicRMWInst::Or, OpAtomicOr);
add(llvm::AtomicRMWInst::Xor, OpAtomicXor);
add(llvm::AtomicRMWInst::Max, OpAtomicSMax);
add(llvm::AtomicRMWInst::Min, OpAtomicSMin);
add(llvm::AtomicRMWInst::UMax, OpAtomicUMax);
add(llvm::AtomicRMWInst::UMin, OpAtomicUMin);
add(llvm::AtomicRMWInst::FAdd, OpAtomicFAddEXT);
add(llvm::AtomicRMWInst::FMin, OpAtomicFMinEXT);
add(llvm::AtomicRMWInst::FMax, OpAtomicFMaxEXT);
}
} // namespace SPIRV
///////////////////////////////////////////////////////////////////////////////
//
// Functions for getting builtin call info
//
///////////////////////////////////////////////////////////////////////////////
namespace OCLUtil {
AtomicWorkItemFenceLiterals getAtomicWorkItemFenceLiterals(CallInst *CI) {
return std::make_tuple(getArgAsInt(CI, 0),
static_cast<OCLMemOrderKind>(getArgAsInt(CI, 1)),
static_cast<OCLScopeKind>(getArgAsInt(CI, 2)));
}
size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name) {
if (Name.starts_with("atomic_compare_exchange"))
return 2;
return 1;
}
size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC) {
if (OC == OpAtomicCompareExchange || OC == OpAtomicCompareExchangeWeak)
return 2;
return 1;
}
// atomic_fetch_[add, sub, min, max] and atomic_fetch_[add, sub, min,
// max]_explicit functions declared in clang headers should be translated
// to corresponding FP-typed Atomic Instructions
bool isComputeAtomicOCLBuiltin(StringRef DemangledName) {
if (!DemangledName.starts_with(kOCLBuiltinName::AtomicPrefix) &&
!DemangledName.starts_with(kOCLBuiltinName::AtomPrefix))
return false;
return llvm::StringSwitch<bool>(DemangledName)
.EndsWith("atomic_add", true)
.EndsWith("atomic_sub", true)
.EndsWith("atomic_min", true)
.EndsWith("atomic_max", true)
.EndsWith("atom_add", true)
.EndsWith("atom_sub", true)
.EndsWith("atom_min", true)
.EndsWith("atom_max", true)
.EndsWith("inc", true)
.EndsWith("dec", true)
.EndsWith("cmpxchg", true)
.EndsWith("and", true)
.EndsWith("or", true)
.EndsWith("xor", true)
.EndsWith("or_explicit", true)
.EndsWith("xor_explicit", true)
.EndsWith("and_explicit", true)
.Default(false);
}
BarrierLiterals getBarrierLiterals(CallInst *CI) {
auto N = CI->arg_size();
assert(N == 1 || N == 2);
StringRef DemangledName;
assert(CI->getCalledFunction() && "Unexpected indirect call");
if (!oclIsBuiltin(CI->getCalledFunction()->getName(), DemangledName)) {
assert(0 &&
"call must a builtin (work_group_barrier or sub_group_barrier)");
}
OCLScopeKind Scope = OCLMS_work_group;
if (DemangledName == kOCLBuiltinName::SubGroupBarrier) {
Scope = OCLMS_sub_group;
}
return std::make_tuple(getArgAsInt(CI, 0),
N == 1 ? OCLMS_work_group
: static_cast<OCLScopeKind>(getArgAsInt(CI, 1)),
Scope);
}
unsigned getExtOp(StringRef OrigName, StringRef GivenDemangledName) {
std::string DemangledName{GivenDemangledName};
if (DemangledName.empty() || !oclIsBuiltin(OrigName, GivenDemangledName))
return ~0U;
LLVM_DEBUG(dbgs() << "getExtOp: demangled name: " << DemangledName << '\n');
OCLExtOpKind EOC;
bool Found = OCLExtOpMap::rfind(DemangledName, &EOC);
if (!Found) {
std::string Prefix;
switch (lastFuncParamType(OrigName)) {
case ParamType::UNSIGNED:
Prefix = "u_";
break;
case ParamType::SIGNED:
Prefix = "s_";
break;
case ParamType::FLOAT:
Prefix = "f";
break;
case ParamType::UNKNOWN:
break;
}
Found = OCLExtOpMap::rfind(Prefix + DemangledName, &EOC);
}
if (Found)
return EOC;
else
return ~0U;
}
///////////////////////////////////////////////////////////////////////////////
//
// Functions for getting module info
//
///////////////////////////////////////////////////////////////////////////////
unsigned encodeOCLVer(unsigned short Major, unsigned char Minor,
unsigned char Rev) {
return (Major * 100 + Minor) * 1000 + Rev;
}
std::tuple<unsigned short, unsigned char, unsigned char>
decodeOCLVer(unsigned Ver) {
unsigned short Major = Ver / 100000;
unsigned char Minor = (Ver % 100000) / 1000;
unsigned char Rev = Ver % 1000;
return std::make_tuple(Major, Minor, Rev);
}
unsigned getOCLVersion(Module *M, bool AllowMulti) {
NamedMDNode *NamedMDOCLVer = M->getNamedMetadata(kSPIR2MD::OCLVer);
NamedMDNode *NamedMDOCLCXXVer = M->getNamedMetadata(kSPIR2MD::OCLCXXVer);
if (!NamedMDOCLVer && !NamedMDOCLCXXVer)
return 0;
// If the module was linked with another module, there may be multiple
// operands.
auto GetVerPair = [](unsigned I, NamedMDNode *NamedMD) {
auto *MD = NamedMD->getOperand(I);
return std::make_pair(getMDOperandAsInt(MD, 0), getMDOperandAsInt(MD, 1));
};
auto GetVer = [=](NamedMDNode *NamedMD) {
assert(NamedMD->getNumOperands() && "Invalid SPIR");
if (!AllowMulti && NamedMD->getNumOperands() != 1)
report_fatal_error(
llvm::Twine("Multiple OCL version metadata not allowed"));
auto Ver = GetVerPair(0, NamedMD);
for (unsigned I = 1, E = NamedMD->getNumOperands(); I != E; ++I)
if (Ver != GetVerPair(I, NamedMD))
report_fatal_error(llvm::Twine("OCL version mismatch"));
return encodeOCLVer(Ver.first, Ver.second, 0);
};
unsigned OCLVer = NamedMDOCLVer ? GetVer(NamedMDOCLVer) : 0;
unsigned OCLCXXVer = NamedMDOCLCXXVer ? GetVer(NamedMDOCLCXXVer) : 0;
// Check if OCLCXXVer is compatible with OCLVer
if (OCLVer && OCLCXXVer) {
if ((OCLVer == kOCLVer::CL20 && OCLCXXVer == kOCLVer::CLCXX10) ||
(OCLVer == kOCLVer::CL30 && OCLCXXVer == kOCLVer::CLCXX2021))
return OCLCXXVer;
report_fatal_error(llvm::Twine(
"opencl cxx version is not compatible with opencl c version!"));
}
return OCLVer;
}
SmallVector<unsigned, 3> decodeMDNode(MDNode *N) {
if (N == NULL)
return {};
size_t NumOperands = N->getNumOperands();
SmallVector<unsigned, 3> ReadVals;
ReadVals.reserve(NumOperands);
for (unsigned I = 0; I < NumOperands; ++I)
ReadVals.push_back(getMDOperandAsInt(N, I));
return ReadVals;
}
/// Encode LLVM type by SPIR-V execution mode VecTypeHint
unsigned encodeVecTypeHint(Type *Ty) {
if (Ty->isHalfTy())
return 4;
if (Ty->isFloatTy())
return 5;
if (Ty->isDoubleTy())
return 6;
if (IntegerType *IntTy = dyn_cast<IntegerType>(Ty)) {
switch (IntTy->getIntegerBitWidth()) {
case 8:
return 0;
case 16:
return 1;
case 32:
return 2;
case 64:
return 3;
default:
llvm_unreachable("invalid integer type");
}
}
if (FixedVectorType *VecTy = dyn_cast<FixedVectorType>(Ty)) {
Type *EleTy = VecTy->getElementType();
unsigned Size = VecTy->getNumElements();
return Size << 16 | encodeVecTypeHint(EleTy);
}
llvm_unreachable("invalid type");
return ~0U;
}
Type *decodeVecTypeHint(LLVMContext &C, unsigned Code) {
unsigned VecWidth = Code >> 16;
unsigned Scalar = Code & 0xFFFF;
Type *ST = nullptr;
switch (Scalar) {
case 0:
case 1:
case 2:
case 3:
ST = IntegerType::get(C, 1 << (3 + Scalar));
break;
case 4:
ST = Type::getHalfTy(C);
break;
case 5:
ST = Type::getFloatTy(C);
break;
case 6:
ST = Type::getDoubleTy(C);
break;
default:
llvm_unreachable("Invalid vec type hint");
return nullptr;
}
if (VecWidth < 1)
return ST;
return FixedVectorType::get(ST, VecWidth);
}
unsigned transVecTypeHint(MDNode *Node) {
return encodeVecTypeHint(getMDOperandAsType(Node, 0));
}
SPIRAddressSpace getOCLOpaqueTypeAddrSpace(Op OpCode) {
switch ((unsigned)OpCode) {
case OpTypeQueue:
return SPIRV_QUEUE_T_ADDR_SPACE;
case OpTypeEvent:
return SPIRV_EVENT_T_ADDR_SPACE;
case OpTypeDeviceEvent:
return SPIRV_CLK_EVENT_T_ADDR_SPACE;
case OpTypeReserveId:
return SPIRV_RESERVE_ID_T_ADDR_SPACE;
case OpTypePipe:
case OpTypePipeStorage:
return SPIRV_PIPE_ADDR_SPACE;
case OpTypeImage:
case OpTypeSampledImage:
case OpTypeVmeImageINTEL:
return SPIRV_IMAGE_ADDR_SPACE;
case OpConstantSampler:
case OpTypeSampler:
return SPIRV_SAMPLER_T_ADDR_SPACE;
case internal::OpTypeJointMatrixINTEL:
case internal::OpTypeJointMatrixINTELv2:
case OpTypeCooperativeMatrixKHR:
case internal::OpTypeTaskSequenceINTEL:
return SPIRAS_Global;
default:
if (isSubgroupAvcINTELTypeOpCode(OpCode))
return SPIRV_AVC_INTEL_T_ADDR_SPACE;
assert(false && "No address space is determined for some OCL type");
return SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE;
}
}
static SPIR::TypeAttributeEnum mapAddrSpaceEnums(SPIRAddressSpace Addrspace) {
switch (Addrspace) {
case SPIRAS_Private:
return SPIR::ATTR_PRIVATE;
case SPIRAS_Global:
return SPIR::ATTR_GLOBAL;
case SPIRAS_Constant:
return SPIR::ATTR_CONSTANT;
case SPIRAS_Local:
return SPIR::ATTR_LOCAL;
case SPIRAS_Generic:
return SPIR::ATTR_GENERIC;
case SPIRAS_GlobalDevice:
return SPIR::ATTR_GLOBAL_DEVICE;
case SPIRAS_GlobalHost:
return SPIR::ATTR_GLOBAL_HOST;
default:
llvm_unreachable("Invalid addrspace enum member");
}
return SPIR::ATTR_NONE;
}
SPIR::TypeAttributeEnum
getOCLOpaqueTypeAddrSpace(SPIR::TypePrimitiveEnum Prim) {
switch (Prim) {
case SPIR::PRIMITIVE_QUEUE_T:
return mapAddrSpaceEnums(SPIRV_QUEUE_T_ADDR_SPACE);
case SPIR::PRIMITIVE_EVENT_T:
return mapAddrSpaceEnums(SPIRV_EVENT_T_ADDR_SPACE);
case SPIR::PRIMITIVE_CLK_EVENT_T:
return mapAddrSpaceEnums(SPIRV_CLK_EVENT_T_ADDR_SPACE);
case SPIR::PRIMITIVE_RESERVE_ID_T:
return mapAddrSpaceEnums(SPIRV_RESERVE_ID_T_ADDR_SPACE);
case SPIR::PRIMITIVE_PIPE_RO_T:
case SPIR::PRIMITIVE_PIPE_WO_T:
return mapAddrSpaceEnums(SPIRV_PIPE_ADDR_SPACE);
case SPIR::PRIMITIVE_IMAGE1D_RO_T:
case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RO_T:
case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T:
case SPIR::PRIMITIVE_IMAGE3D_RO_T:
case SPIR::PRIMITIVE_IMAGE1D_WO_T:
case SPIR::PRIMITIVE_IMAGE1D_ARRAY_WO_T:
case SPIR::PRIMITIVE_IMAGE1D_BUFFER_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_DEPTH_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T:
case SPIR::PRIMITIVE_IMAGE3D_WO_T:
case SPIR::PRIMITIVE_IMAGE1D_RW_T:
case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RW_T:
case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T:
case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T:
case SPIR::PRIMITIVE_IMAGE3D_RW_T:
return mapAddrSpaceEnums(SPIRV_IMAGE_ADDR_SPACE);
default:
llvm_unreachable("No address space is determined for a SPIR primitive");
}
return SPIR::ATTR_NONE;
}
// Fetch type of invoke function passed to device execution built-ins
static FunctionType *getBlockInvokeTy(Function *F, unsigned BlockIdx) {
auto Params = F->getFunctionType()->params();
PointerType *FuncPtr = cast<PointerType>(Params[BlockIdx]);
return FunctionType::get(FuncPtr, Params, false);
}
class OCLBuiltinFuncMangleInfo : public SPIRV::BuiltinFuncMangleInfo {
public:
OCLBuiltinFuncMangleInfo(Function *F) : F(F) {}
OCLBuiltinFuncMangleInfo() = default;
void init(StringRef UniqName) override {
// Make a local copy as we will modify the string in init function
std::string TempStorage = UniqName.str();
auto NameRef = StringRef(TempStorage);
// Helper functions to erase substrings from NameRef (i.e. TempStorage)
auto EraseSubstring = [&NameRef, &TempStorage](const std::string &ToErase) {
size_t Pos = TempStorage.find(ToErase);
if (Pos != std::string::npos) {
TempStorage.erase(Pos, ToErase.length());
// re-take StringRef as TempStorage was updated
NameRef = StringRef(TempStorage);
}
};
auto EraseSymbol = [&NameRef, &TempStorage](size_t Index) {
TempStorage.erase(Index, 1);
// re-take StringRef as TempStorage was updated
NameRef = StringRef(TempStorage);
};
if (NameRef.starts_with("async_work_group")) {
addUnsignedArg(-1);
setArgAttr(1, SPIR::ATTR_CONST);
} else if (NameRef.starts_with("printf"))
setVarArg(1);
else if (NameRef.starts_with("write_imageui"))
addUnsignedArg(2);
else if (NameRef == "prefetch") {
addUnsignedArg(1);
setArgAttr(0, SPIR::ATTR_CONST);
} else if (NameRef == "get_kernel_work_group_size" ||
NameRef == "get_kernel_preferred_work_group_size_multiple") {
assert(F && "lack of necessary information");
const size_t BlockArgIdx = 0;
FunctionType *InvokeTy = getBlockInvokeTy(F, BlockArgIdx);
if (InvokeTy->getNumParams() > 1)
setLocalArgBlock(BlockArgIdx);
} else if (NameRef.starts_with("__enqueue_kernel")) {
// clang doesn't mangle enqueue_kernel builtins
setAsDontMangle();
} else if (NameRef.starts_with("get_") || NameRef == "nan" ||
NameRef == "mem_fence" || NameRef.starts_with("shuffle")) {
addUnsignedArg(-1);
if (NameRef.starts_with(kOCLBuiltinName::GetFence)) {
setArgAttr(0, SPIR::ATTR_CONST);
addVoidPtrArg(0);
}
} else if (NameRef.contains("barrier")) {
addUnsignedArg(0);
if (NameRef == "work_group_barrier" || NameRef == "sub_group_barrier" ||
NameRef == "intel_work_group_barrier_arrive" ||
NameRef == "intel_work_group_barrier_wait")
setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE);
} else if (NameRef.starts_with("atomic_work_item_fence")) {
addUnsignedArg(0);
setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER);
setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE);
} else if (NameRef.starts_with("atom_")) {
setArgAttr(0, SPIR::ATTR_VOLATILE);
if (NameRef.ends_with("_umax") || NameRef.ends_with("_umin")) {
addUnsignedArg(-1);
// We need to remove u to match OpenCL C built-in function name
EraseSymbol(5);
}
} else if (NameRef.starts_with("atomic")) {
setArgAttr(0, SPIR::ATTR_VOLATILE);
if (NameRef.contains("_umax") || NameRef.contains("_umin")) {
addUnsignedArg(-1);
// We need to remove u to match OpenCL C built-in function name
if (NameRef.contains("_fetch"))
EraseSymbol(13);
else
EraseSymbol(7);
}
if (NameRef.contains("store_explicit") ||
NameRef.contains("exchange_explicit") ||
(NameRef.starts_with("atomic_fetch") &&
NameRef.contains("explicit"))) {
setEnumArg(2, SPIR::PRIMITIVE_MEMORY_ORDER);
setEnumArg(3, SPIR::PRIMITIVE_MEMORY_SCOPE);
} else if (NameRef.contains("load_explicit") ||
(NameRef.starts_with("atomic_flag") &&
NameRef.contains("explicit"))) {
setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER);
setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE);
} else if (NameRef.ends_with("compare_exchange_strong_explicit") ||
NameRef.ends_with("compare_exchange_weak_explicit")) {
setEnumArg(3, SPIR::PRIMITIVE_MEMORY_ORDER);
setEnumArg(4, SPIR::PRIMITIVE_MEMORY_ORDER);
setEnumArg(5, SPIR::PRIMITIVE_MEMORY_SCOPE);
}
// Don't set atomic property to the first argument of 1.2 atomic
// built-ins.
if (!NameRef.ends_with("xchg") && // covers _cmpxchg too
(NameRef.contains("fetch") ||
!(NameRef.ends_with("_add") || NameRef.ends_with("_sub") ||
NameRef.ends_with("_inc") || NameRef.ends_with("_dec") ||
NameRef.ends_with("_min") || NameRef.ends_with("_max") ||
NameRef.ends_with("_and") || NameRef.ends_with("_or") ||
NameRef.ends_with("_xor")))) {
addAtomicArg(0);
}
} else if (NameRef.starts_with("uconvert_")) {
addUnsignedArg(0);
NameRef = NameRef.drop_front(1);
UnmangledName.erase(0, 1);
} else if (NameRef.starts_with("s_")) {
if (NameRef == "s_upsample")
addUnsignedArg(1);
NameRef = NameRef.drop_front(2);
} else if (NameRef.starts_with("u_")) {
addUnsignedArg(-1);
NameRef = NameRef.drop_front(2);
} else if (NameRef == "fclamp") {
NameRef = NameRef.drop_front(1);
}
// handle [read|write]pipe builtins (plus two i32 literal args
// required by SPIR 2.0 provisional specification):
else if (NameRef == "read_pipe_2" || NameRef == "write_pipe_2") {
// with 2 arguments (plus two i32 literals):
// int read_pipe (read_only pipe gentype p, gentype *ptr)
// int write_pipe (write_only pipe gentype p, const gentype *ptr)
addVoidPtrArg(1);
addUnsignedArg(2);
addUnsignedArg(3);
// OpenCL-like representation of blocking pipes
} else if (NameRef == "read_pipe_2_bl" || NameRef == "write_pipe_2_bl") {
// with 2 arguments (plus two i32 literals):
// int read_pipe_bl (read_only pipe gentype p, gentype *ptr)
// int write_pipe_bl (write_only pipe gentype p, const gentype *ptr)
addVoidPtrArg(1);
addUnsignedArg(2);
addUnsignedArg(3);
} else if (NameRef == "read_pipe_4" || NameRef == "write_pipe_4") {
// with 4 arguments (plus two i32 literals):
// int read_pipe (read_only pipe gentype p, reserve_id_t reserve_id, uint
// index, gentype *ptr) int write_pipe (write_only pipe gentype p,
// reserve_id_t reserve_id, uint index, const gentype *ptr)
addUnsignedArg(2);
addVoidPtrArg(3);
addUnsignedArg(4);
addUnsignedArg(5);
} else if (NameRef.contains("reserve_read_pipe") ||
NameRef.contains("reserve_write_pipe")) {
// process [|work_group|sub_group]reserve[read|write]pipe builtins
addUnsignedArg(1);
addUnsignedArg(2);
addUnsignedArg(3);
} else if (NameRef.contains("commit_read_pipe") ||
NameRef.contains("commit_write_pipe")) {
// process [|work_group|sub_group]commit[read|write]pipe builtins
addUnsignedArg(2);
addUnsignedArg(3);
} else if (NameRef == "capture_event_profiling_info") {
addVoidPtrArg(2);
setEnumArg(1, SPIR::PRIMITIVE_CLK_PROFILING_INFO);
} else if (NameRef == "enqueue_marker") {
setArgAttr(2, SPIR::ATTR_CONST);
addUnsignedArg(1);
} else if (NameRef.starts_with("vload")) {
addUnsignedArg(0);
setArgAttr(1, SPIR::ATTR_CONST);
} else if (NameRef.starts_with("vstore")) {
addUnsignedArg(1);
} else if (NameRef.starts_with("ndrange_")) {
addUnsignedArgs(0, 2);
if (NameRef[8] == '2' || NameRef[8] == '3') {
setArgAttr(0, SPIR::ATTR_CONST);
setArgAttr(1, SPIR::ATTR_CONST);
setArgAttr(2, SPIR::ATTR_CONST);
}
} else if (NameRef.contains("umax")) {
addUnsignedArg(-1);
EraseSymbol(NameRef.find("umax"));
} else if (NameRef.contains("umin")) {
addUnsignedArg(-1);
EraseSymbol(NameRef.find("umin"));
} else if (NameRef.contains("broadcast")) {
addUnsignedArg(-1);
} else if (NameRef.starts_with(kOCLBuiltinName::SampledReadImage)) {
if (!NameRef.consume_front(kOCLBuiltinName::Sampled))
report_fatal_error(llvm::Twine("Builtin name illformed"));
addSamplerArg(1);
} else if (NameRef.contains(kOCLSubgroupsAVCIntel::Prefix)) {
if (NameRef.contains("evaluate_ipe"))
addSamplerArg(1);
else if (NameRef.contains("evaluate_with_single_reference"))
addSamplerArg(2);
else if (NameRef.contains("evaluate_with_multi_reference")) {
addUnsignedArg(1);
std::string PostFix = "_interlaced";
if (NameRef.contains(PostFix)) {
addUnsignedArg(2);
addSamplerArg(3);
EraseSubstring(PostFix);
} else
addSamplerArg(2);
} else if (NameRef.contains("evaluate_with_dual_reference"))
addSamplerArg(3);
else if (NameRef.contains("fme_initialize"))
addUnsignedArgs(0, 6);
else if (NameRef.contains("bme_initialize"))
addUnsignedArgs(0, 7);
else if (NameRef.contains("set_inter_base_multi_reference_penalty") ||
NameRef.contains("set_inter_shape_penalty") ||
NameRef.contains("set_inter_direction_penalty"))
addUnsignedArg(0);
else if (NameRef.contains("set_motion_vector_cost_function"))
addUnsignedArgs(0, 2);
else if (NameRef.contains("interlaced_field_polarity"))
addUnsignedArg(0);
else if (NameRef.contains("interlaced_field_polarities"))
addUnsignedArgs(0, 1);
else if (NameRef.contains(kOCLSubgroupsAVCIntel::MCEPrefix)) {
if (NameRef.contains("get_default"))
addUnsignedArgs(0, 1);
} else if (NameRef.contains(kOCLSubgroupsAVCIntel::IMEPrefix)) {
if (NameRef.contains("initialize"))
addUnsignedArgs(0, 2);
else if (NameRef.contains("set_single_reference"))
addUnsignedArg(1);
else if (NameRef.contains("set_dual_reference"))
addUnsignedArg(2);
else if (NameRef.contains("set_weighted_sad") ||
NameRef.contains("set_early_search_termination_threshold"))
addUnsignedArg(0);
else if (NameRef.contains("adjust_ref_offset"))
addUnsignedArgs(1, 3);
else if (NameRef.contains("set_max_motion_vector_count") ||
NameRef.contains("get_border_reached"))
addUnsignedArg(0);
else if (NameRef.contains("shape_distortions") ||
NameRef.contains("shape_motion_vectors") ||
NameRef.contains("shape_reference_ids")) {
if (NameRef.contains("single_reference")) {
addUnsignedArg(1);
EraseSubstring("_single_reference");
} else if (NameRef.contains("dual_reference")) {
addUnsignedArgs(1, 2);
EraseSubstring("_dual_reference");
}
} else if (NameRef.contains("ref_window_size"))
addUnsignedArg(0);
} else if (NameRef.contains(kOCLSubgroupsAVCIntel::SICPrefix)) {
if (NameRef.contains("initialize") ||
NameRef.contains("set_intra_luma_shape_penalty"))
addUnsignedArg(0);
else if (NameRef.contains("configure_ipe")) {
if (NameRef.contains("_luma")) {
addUnsignedArgs(0, 6);
EraseSubstring("_luma");
}
if (NameRef.contains("_chroma")) {
addUnsignedArgs(7, 9);
EraseSubstring("_chroma");
}
} else if (NameRef.contains("configure_skc"))
addUnsignedArgs(0, 4);
else if (NameRef.contains("set_skc")) {
if (NameRef.contains("forward_transform_enable"))
addUnsignedArg(0);
} else if (NameRef.contains("set_block")) {
if (NameRef.contains("based_raw_skip_sad"))
addUnsignedArg(0);
} else if (NameRef.contains("get_motion_vector_mask")) {
addUnsignedArgs(0, 1);
} else if (NameRef.contains("luma_mode_cost_function"))
addUnsignedArgs(0, 2);
else if (NameRef.contains("chroma_mode_cost_function"))
addUnsignedArg(0);
}
} else if (NameRef.starts_with("intel_sub_group_shuffle")) {
if (NameRef.ends_with("_down") || NameRef.ends_with("_up"))
addUnsignedArg(2);
else
addUnsignedArg(1);
} else if (NameRef.starts_with("intel_sub_group_block_write")) {
// distinguish write to image and other data types based on number of
// arguments--images have one more argument.
if (F->getFunctionType()->getNumParams() == 2) {
addUnsignedArg(0);
addUnsignedArg(1);
} else {
addUnsignedArg(2);
}
} else if (NameRef.starts_with("intel_sub_group_block_read")) {
// distinguish read from image and other data types based on number of
// arguments--images have one more argument.
if (F->getFunctionType()->getNumParams() == 1) {
setArgAttr(0, SPIR::ATTR_CONST);
addUnsignedArg(0);
}
} else if (NameRef.starts_with("intel_sub_group_media_block_write")) {
addUnsignedArg(3);
} else if (NameRef.starts_with(kOCLBuiltinName::SubGroupPrefix)) {
if (NameRef.contains("ballot")) {
if (NameRef.contains("inverse") || NameRef.contains("bit_count") ||
NameRef.contains("inclusive_scan") ||
NameRef.contains("exclusive_scan") ||
NameRef.contains("find_lsb") || NameRef.contains("find_msb"))
addUnsignedArg(0);
else if (NameRef.contains("bit_extract")) {
addUnsignedArgs(0, 1);
}
} else if (NameRef.starts_with("sub_group_clustered_rotate")) {
addUnsignedArg(2);
} else if (NameRef.contains("shuffle") || NameRef.contains("clustered"))
addUnsignedArg(1);
} else if (NameRef.starts_with("bitfield_insert")) {
addUnsignedArgs(2, 3);
} else if (NameRef.starts_with("bitfield_extract_signed") ||
NameRef.starts_with("bitfield_extract_unsigned")) {
addUnsignedArgs(1, 2);
} else if (NameRef.starts_with("dot_")) {
if (NameRef.contains("4x8packed")) {
addUnsignedArgs(0, 1);
if (NameRef == "dot_acc_sat_4x8packed_uu_uint")
addUnsignedArg(2);
} else {
if (NameRef.ends_with("_uu")) {
addUnsignedArgs(0, 1);
if (NameRef.starts_with("dot_acc_sat"))
addUnsignedArg(2);
} else if (NameRef.ends_with("_su"))
addUnsignedArg(1);
NameRef = NameRef.drop_back(std::string("_uu").length());
}
}
// Store the final version of a function name
UnmangledName = NameRef.str();
}
// Auxiliarry information, it is expected that it is relevant at the moment
// the init method is called.
Function *F; // SPIRV decorated function
};
std::unique_ptr<SPIRV::BuiltinFuncMangleInfo> makeMangler(Function &F) {
return std::make_unique<OCLBuiltinFuncMangleInfo>(&F);
}
bool isSamplerTy(Type *Ty) {
if (auto *TPT = dyn_cast_or_null<TypedPointerType>(Ty)) {
auto *STy = dyn_cast_or_null<StructType>(TPT->getElementType());
return STy && STy->hasName() && STy->getName() == kSPR2TypeName::Sampler;
}
if (auto *TET = dyn_cast_or_null<TargetExtType>(Ty)) {
return TET->getName() == "spirv.Sampler";
}
return false;
}
bool isPipeOrAddressSpaceCastBI(const StringRef MangledName) {
return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" ||
MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" ||
MangledName == "write_pipe_4" || MangledName == "read_pipe_4" ||
MangledName == "reserve_write_pipe" ||
MangledName == "reserve_read_pipe" ||
MangledName == "commit_write_pipe" ||
MangledName == "commit_read_pipe" ||
MangledName == "work_group_reserve_write_pipe" ||
MangledName == "work_group_reserve_read_pipe" ||
MangledName == "work_group_commit_write_pipe" ||
MangledName == "work_group_commit_read_pipe" ||
MangledName == "get_pipe_num_packets_ro" ||
MangledName == "get_pipe_max_packets_ro" ||
MangledName == "get_pipe_num_packets_wo" ||
MangledName == "get_pipe_max_packets_wo" ||
MangledName == "sub_group_reserve_write_pipe" ||
MangledName == "sub_group_reserve_read_pipe" ||
MangledName == "sub_group_commit_write_pipe" ||
MangledName == "sub_group_commit_read_pipe" ||
MangledName == "to_global" || MangledName == "to_local" ||
MangledName == "to_private";
}
bool isEnqueueKernelBI(const StringRef MangledName) {
return MangledName == "__enqueue_kernel_basic" ||
MangledName == "__enqueue_kernel_basic_events" ||
MangledName == "__enqueue_kernel_varargs" ||
MangledName == "__enqueue_kernel_events_varargs";
}
bool isKernelQueryBI(const StringRef MangledName) {
return MangledName == "__get_kernel_work_group_size_impl" ||
MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
}
// isUnfusedMulAdd checks if we have the following (most common for fp
// contranction) pattern in LLVM IR:
//
// %mul = fmul float %a, %b
// %add = fadd float %mul, %c
//
// This pattern indicates that fp contraction could have been disabled by
// #pragma OPENCL FP_CONTRACT OFF. When contraction is enabled (by a pragma or
// by clang's -ffp-contract=fast), clang would generate:
//
// %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
//
// or
//
// %mul = fmul contract float %a, %b
// %add = fadd contract float %mul, %c
//
// Note that optimizations may form an unfused fmuladd from fadd+load or
// fadd+call, so this check is quite restrictive (see the comment below).
//
bool isUnfusedMulAdd(BinaryOperator *B) {
if (B->getOpcode() != Instruction::FAdd &&
B->getOpcode() != Instruction::FSub)
return false;
if (B->hasAllowContract()) {
// If this fadd or fsub itself has a contract flag, the operation can be
// contracted regardless of the operands.
return false;
}
// Otherwise, we cannot easily tell if the operation can be a candidate for
// contraction or not. Consider the following cases:
//
// %mul = alloca float
// %t1 = fmul float %a, %b
// store float* %mul, float %t
// %t2 = load %mul
// %r = fadd float %t2, %c
//
// LLVM IR does not allow %r to be contracted. However, after an optimization
// it becomes a candidate for contraction if ContractionOFF is not set in
// SPIR-V:
//
// %t1 = fmul float %a, %b
// %r = fadd float %t1, %c
//
// To be on a safe side, we disallow everything that is even remotely similar
// to fmul + fadd.
return true;
}
std::string getIntelSubgroupBlockDataPostfix(unsigned ElementBitSize,
unsigned VectorNumElements) {
std::ostringstream OSS;
switch (ElementBitSize) {
case 8:
OSS << "_uc";
break;
case 16:
OSS << "_us";
break;
case 32:
// Intentionally does nothing since _ui variant is only an alias.
break;
case 64:
OSS << "_ul";
break;
default:
llvm_unreachable(
"Incorrect data bitsize for intel_subgroup_block builtins");
}
switch (VectorNumElements) {
case 1:
break;
case 2:
case 4:
case 8:
OSS << VectorNumElements;
break;
case 16:
assert((ElementBitSize == 8 || ElementBitSize == 16) &&
"16 elements vector allowed only for char and short builtins");
OSS << VectorNumElements;
break;
default:
llvm_unreachable(
"Incorrect vector length for intel_subgroup_block builtins");
}
return OSS.str();
}
void insertImageNameAccessQualifier(SPIRVAccessQualifierKind Acc,
std::string &Name) {
std::string QName = rmap<std::string>(Acc);
// transform: read_only -> ro, write_only -> wo, read_write -> rw
QName = QName.substr(0, 1) + QName.substr(QName.find("_") + 1, 1) + "_";
assert(!Name.empty() && "image name should not be empty");
Name.insert(Name.size() - 1, QName);
}
} // namespace OCLUtil
Value *SPIRV::transOCLMemScopeIntoSPIRVScope(Value *MemScope,
std::optional<int> DefaultCase,
Instruction *InsertBefore) {
if (auto *C = dyn_cast<ConstantInt>(MemScope)) {
return ConstantInt::get(
C->getType(), map<Scope>(static_cast<OCLScopeKind>(C->getZExtValue())));
}
// If memory_scope is not a constant, then we have to insert dynamic mapping:
return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemScope, MemScope,
OCLMemScopeMap::getMap(), /* IsReverse */ false,
DefaultCase, InsertBefore);
}
Value *
SPIRV::transOCLMemOrderIntoSPIRVMemorySemantics(Value *MemOrder,
std::optional<int> DefaultCase,
Instruction *InsertBefore) {
if (auto *C = dyn_cast<ConstantInt>(MemOrder)) {
return ConstantInt::get(
C->getType(), mapOCLMemSemanticToSPIRV(
0, static_cast<OCLMemOrderKind>(C->getZExtValue())));
}
return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemOrder, MemOrder,
OCLMemOrderMap::getMap(), /* IsReverse */ false,
DefaultCase, InsertBefore);
}
Value *
SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Value *MemScope,
Instruction *InsertBefore) {
if (auto *C = dyn_cast<ConstantInt>(MemScope)) {
return ConstantInt::get(C->getType(), rmap<OCLScopeKind>(static_cast<Scope>(
C->getZExtValue())));
}
if (auto *CI = dyn_cast<CallInst>(MemScope)) {
Function *F = CI->getCalledFunction();
if (F && F->getName() == kSPIRVName::TranslateOCLMemScope) {
// In case the SPIR-V module was created from an OpenCL program by
// *this* SPIR-V generator, we know that the value passed to
// __translate_ocl_memory_scope is what we should pass to the
// OpenCL builtin now.
return CI->getArgOperand(0);
}
}
return getOrCreateSwitchFunc(
kSPIRVName::TranslateSPIRVMemScope, MemScope, OCLMemScopeMap::getRMap(),
/* IsReverse */ true, std::nullopt, InsertBefore);
}
Value *
SPIRV::transSPIRVMemorySemanticsIntoOCLMemoryOrder(Value *MemorySemantics,
Instruction *InsertBefore) {
if (auto *C = dyn_cast<ConstantInt>(MemorySemantics)) {
return ConstantInt::get(C->getType(),
mapSPIRVMemSemanticToOCL(C->getZExtValue()).second);
}
if (auto *CI = dyn_cast<CallInst>(MemorySemantics)) {
Function *F = CI->getCalledFunction();
if (F && F->getName() == kSPIRVName::TranslateOCLMemOrder) {
// In case the SPIR-V module was created from an OpenCL program by
// *this* SPIR-V generator, we know that the value passed to
// __translate_ocl_memory_order is what we should pass to the
// OpenCL builtin now.
return CI->getArgOperand(0);
}
}
// SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and
// therefore, we need to apply mask
int Mask = MemorySemanticsMaskNone | MemorySemanticsAcquireMask |
MemorySemanticsReleaseMask | MemorySemanticsAcquireReleaseMask |
MemorySemanticsSequentiallyConsistentMask;
return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemOrder,
MemorySemantics, OCLMemOrderMap::getRMap(),
/* IsReverse */ true, std::nullopt, InsertBefore,
Mask);
}
Value *SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(
Value *MemorySemantics, Instruction *InsertBefore) {
if (auto *C = dyn_cast<ConstantInt>(MemorySemantics)) {
return ConstantInt::get(C->getType(),
mapSPIRVMemSemanticToOCL(C->getZExtValue()).first);
}
// TODO: any possible optimizations?
// SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and
// therefore, we need to apply mask
int Mask = MemorySemanticsWorkgroupMemoryMask |
MemorySemanticsCrossWorkgroupMemoryMask |
MemorySemanticsImageMemoryMask;
return getOrCreateSwitchFunc(
kSPIRVName::TranslateSPIRVMemFence, MemorySemantics,
OCLMemFenceExtendedMap::getRMap(),
/* IsReverse */ true, std::nullopt, InsertBefore, Mask);
}
void llvm::mangleOpenClBuiltin(const std::string &UniqName,
ArrayRef<Type *> ArgTypes,
std::string &MangledName) {
OCLUtil::OCLBuiltinFuncMangleInfo BtnInfo;
MangledName = SPIRV::mangleBuiltin(UniqName, ArgTypes, &BtnInfo);
}
Reference in New Issue View Git Blame Copy Permalink