39 #include "util_macro.cuh"
41 #include "util_namespace.cuh"
64 template <
bool IF,
typename ThenType,
typename ElseType>
71 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
73 template <
typename ThenType,
typename ElseType>
74 struct If<false, ThenType, ElseType>
76 typedef ElseType
Type;
79 #endif // DOXYGEN_SHOULD_SKIP_THIS
90 template <
typename A,
typename B>
99 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
101 template <
typename A>
110 #endif // DOXYGEN_SHOULD_SKIP_THIS
122 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
124 template <
typename T>
125 __host__ __device__ __forceinline__
NullType& operator =(
const T& b) {
return *
this; }
127 __host__ __device__ __forceinline__
bool operator ==(
const NullType& b) {
return true; }
129 __host__ __device__ __forceinline__
bool operator !=(
const NullType& b) {
return false; }
131 #endif // DOXYGEN_SHOULD_SKIP_THIS
145 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
153 template <
typename T>
165 ALIGN_BYTES =
sizeof(Pad) -
sizeof(T)
171 template <>
struct AlignBytes<short4> {
enum { ALIGN_BYTES = 8 }; };
172 template <>
struct AlignBytes<ushort4> {
enum { ALIGN_BYTES = 8 }; };
173 template <>
struct AlignBytes<int2> {
enum { ALIGN_BYTES = 8 }; };
174 template <>
struct AlignBytes<uint2> {
enum { ALIGN_BYTES = 8 }; };
176 template <>
struct AlignBytes<long2> {
enum { ALIGN_BYTES = 8 }; };
177 template <>
struct AlignBytes<ulong2> {
enum { ALIGN_BYTES = 8 }; };
179 template <>
struct AlignBytes<long long> {
enum { ALIGN_BYTES = 8 }; };
180 template <>
struct AlignBytes<unsigned long long> {
enum { ALIGN_BYTES = 8 }; };
181 template <>
struct AlignBytes<float2> {
enum { ALIGN_BYTES = 8 }; };
182 template <>
struct AlignBytes<double> {
enum { ALIGN_BYTES = 8 }; };
184 template <>
struct AlignBytes<int4> {
enum { ALIGN_BYTES = 16 }; };
185 template <>
struct AlignBytes<uint4> {
enum { ALIGN_BYTES = 16 }; };
186 template <>
struct AlignBytes<float4> {
enum { ALIGN_BYTES = 16 }; };
188 template <>
struct AlignBytes<long2> {
enum { ALIGN_BYTES = 16 }; };
189 template <>
struct AlignBytes<ulong2> {
enum { ALIGN_BYTES = 16 }; };
191 template <>
struct AlignBytes<long4> {
enum { ALIGN_BYTES = 16 }; };
192 template <>
struct AlignBytes<ulong4> {
enum { ALIGN_BYTES = 16 }; };
193 template <>
struct AlignBytes<longlong2> {
enum { ALIGN_BYTES = 16 }; };
194 template <>
struct AlignBytes<ulonglong2> {
enum { ALIGN_BYTES = 16 }; };
195 template <>
struct AlignBytes<double2> {
enum { ALIGN_BYTES = 16 }; };
196 template <>
struct AlignBytes<longlong4> {
enum { ALIGN_BYTES = 16 }; };
197 template <>
struct AlignBytes<ulonglong4> {
enum { ALIGN_BYTES = 16 }; };
198 template <>
struct AlignBytes<double4> {
enum { ALIGN_BYTES = 16 }; };
202 template <
typename T>
206 ALIGN_BYTES = AlignBytes<T>::ALIGN_BYTES
209 template <
typename Unit>
213 UNIT_ALIGN_BYTES = AlignBytes<Unit>::ALIGN_BYTES,
214 IS_MULTIPLE = (
sizeof(T) %
sizeof(Unit) == 0) && (ALIGN_BYTES % UNIT_ALIGN_BYTES == 0)
219 typedef typename If<IsMultiple<int>::IS_MULTIPLE,
221 typename If<IsMultiple<short>::IS_MULTIPLE,
223 unsigned char>::Type>::Type ShuffleWord;
226 typedef typename If<IsMultiple<long long>::IS_MULTIPLE,
228 ShuffleWord>::Type VolatileWord;
231 typedef typename If<IsMultiple<longlong2>::IS_MULTIPLE,
233 VolatileWord>::Type DeviceWord;
236 typedef typename If<IsMultiple<int4>::IS_MULTIPLE,
238 typename If<IsMultiple<int2>::IS_MULTIPLE,
240 ShuffleWord>::Type>::Type TextureWord;
246 struct UnitWord <float2>
248 typedef int ShuffleWord;
249 #if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
250 typedef float VolatileWord;
251 typedef uint2 DeviceWord;
253 typedef unsigned long long VolatileWord;
254 typedef unsigned long long DeviceWord;
256 typedef float2 TextureWord;
261 struct UnitWord <float4>
263 typedef int ShuffleWord;
264 #if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
265 typedef float VolatileWord;
266 typedef uint4 DeviceWord;
268 typedef unsigned long long VolatileWord;
269 typedef ulonglong2 DeviceWord;
271 typedef float4 TextureWord;
277 struct UnitWord <char2>
279 typedef unsigned short ShuffleWord;
280 #if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
281 typedef unsigned short VolatileWord;
282 typedef short DeviceWord;
284 typedef unsigned short VolatileWord;
285 typedef unsigned short DeviceWord;
287 typedef unsigned short TextureWord;
290 #endif // DOXYGEN_SHOULD_SKIP_THIS
301 template <
typename T,
int vec_elements>
struct CubVector;
303 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
308 MAX_VEC_ELEMENTS = 4,
315 template <
typename T>
327 template <
typename T>
328 struct CubVector<T, 2>
334 typedef CubVector<T, 2> Type;
340 template <
typename T>
341 struct CubVector<T, 3>
348 typedef CubVector<T, 3> Type;
354 template <
typename T>
355 struct CubVector<T, 4>
363 typedef CubVector<T, 4> Type;
370 #define CUB_DEFINE_VECTOR_TYPE(base_type,short_type) \
372 template<> struct CubVector<base_type, 1> : short_type##1 \
374 typedef base_type BaseType; \
375 typedef short_type##1 Type; \
376 __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const { \
378 retval.x = x + other.x; \
381 __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const { \
383 retval.x = x - other.x; \
388 template<> struct CubVector<base_type, 2> : short_type##2 \
390 typedef base_type BaseType; \
391 typedef short_type##2 Type; \
392 __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const { \
394 retval.x = x + other.x; \
395 retval.y = y + other.y; \
398 __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const { \
400 retval.x = x - other.x; \
401 retval.y = y - other.y; \
406 template<> struct CubVector<base_type, 3> : short_type##3 \
408 typedef base_type BaseType; \
409 typedef short_type##3 Type; \
410 __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const { \
412 retval.x = x + other.x; \
413 retval.y = y + other.y; \
414 retval.z = z + other.z; \
417 __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const { \
419 retval.x = x - other.x; \
420 retval.y = y - other.y; \
421 retval.z = z - other.z; \
426 template<> struct CubVector<base_type, 4> : short_type##4 \
428 typedef base_type BaseType; \
429 typedef short_type##4 Type; \
430 __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const { \
432 retval.x = x + other.x; \
433 retval.y = y + other.y; \
434 retval.z = z + other.z; \
435 retval.w = w + other.w; \
438 __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const { \
440 retval.x = x - other.x; \
441 retval.y = y - other.y; \
442 retval.z = z - other.z; \
443 retval.w = w - other.w; \
451 CUB_DEFINE_VECTOR_TYPE(
char,
char)
452 CUB_DEFINE_VECTOR_TYPE(
signed char,
char)
453 CUB_DEFINE_VECTOR_TYPE(
short,
short)
454 CUB_DEFINE_VECTOR_TYPE(
int,
int)
455 CUB_DEFINE_VECTOR_TYPE(
long,
long)
456 CUB_DEFINE_VECTOR_TYPE(
long long, longlong)
457 CUB_DEFINE_VECTOR_TYPE(
unsigned char, uchar)
458 CUB_DEFINE_VECTOR_TYPE(
unsigned short, ushort)
459 CUB_DEFINE_VECTOR_TYPE(
unsigned int, uint)
460 CUB_DEFINE_VECTOR_TYPE(
unsigned long, ulong)
461 CUB_DEFINE_VECTOR_TYPE(
unsigned long long, ulonglong)
462 CUB_DEFINE_VECTOR_TYPE(
float,
float)
463 CUB_DEFINE_VECTOR_TYPE(
double,
double)
464 CUB_DEFINE_VECTOR_TYPE(
bool, uchar)
467 #undef CUB_DEFINE_VECTOR_TYPE
469 #endif // DOXYGEN_SHOULD_SKIP_THIS
480 template <
typename T>
495 __host__ __device__ __forceinline__ T&
Alias()
497 return reinterpret_cast<T&
>(*this);
505 template <
typename _T,
typename _Offset>
511 #if (CUB_PTX_ARCH == 0)
534 template <
typename _Key,
typename _Value>
551 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
557 template <
typename T>
558 __host__ __device__ __forceinline__ T ZeroInitialize()
560 #if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
562 typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
563 const int MULTIPLE =
sizeof(T) /
sizeof(ShuffleWord);
564 ShuffleWord words[MULTIPLE];
566 for (
int i = 0; i < MULTIPLE; ++i)
568 return *
reinterpret_cast<T*
>(words);
581 template <
typename T,
int COUNT>
588 #endif // DOXYGEN_SHOULD_SKIP_THIS
598 template <
typename T>
642 template <
int N,
int CURRENT_VAL = N,
int COUNT = 0>
649 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
650 template <
int N,
int COUNT>
651 struct Log2<N, 0, COUNT>
653 enum {VALUE = (1 << (COUNT - 1) < N) ?
657 #endif // DOXYGEN_SHOULD_SKIP_THIS
666 enum { VALUE = ((N & (N - 1)) == 0) };
670 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
680 template <
typename Tp>
686 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
688 template <
typename Tp>
689 struct IsPointer<Tp*>
694 #endif // DOXYGEN_SHOULD_SKIP_THIS
705 template <
typename Tp>
711 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
713 template <
typename Tp>
714 struct IsVolatile<Tp volatile>
719 #endif // DOXYGEN_SHOULD_SKIP_THIS
732 template <
typename Tp,
typename Up = Tp>
733 struct RemoveQualifiers
739 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
741 template <
typename Tp,
typename Up>
742 struct RemoveQualifiers<Tp, volatile Up>
747 template <
typename Tp,
typename Up>
748 struct RemoveQualifiers<Tp, const Up>
753 template <
typename Tp,
typename Up>
754 struct RemoveQualifiers<Tp, const volatile Up>
759 #endif // DOXYGEN_SHOULD_SKIP_THIS
771 #define CUB_DEFINE_DETECT_NESTED_TYPE(detector_name, nested_type_name) \
772 template <typename T> \
773 struct detector_name \
775 template <typename C> \
776 static char& test(typename C::nested_type_name*); \
777 template <typename> \
778 static int& test(...); \
781 VALUE = sizeof(test<T>(0)) < sizeof(int) \
794 template <
bool Condition,
class T =
void>
801 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
804 struct EnableIf<false, T> {};
806 #endif // DOXYGEN_SHOULD_SKIP_THIS
816 template <
typename T,
typename BinaryOp>
817 struct BinaryOpHasIdxParam
820 template <
typename BinaryOpT,
bool (BinaryOpT::*)(const T &a, const T &b,
unsigned int idx) const>
struct SFINAE1 {};
821 template <
typename BinaryOpT,
bool (BinaryOpT::*)(const T &a, const T &b,
unsigned int idx)>
struct SFINAE2 {};
822 template <
typename BinaryOpT,
bool (BinaryOpT::*)(T a, T b,
unsigned int idx) const>
struct SFINAE3 {};
823 template <
typename BinaryOpT,
bool (BinaryOpT::*)(T a, T b,
unsigned int idx)>
struct SFINAE4 {};
825 template <
typename BinaryOpT,
bool (BinaryOpT::*)(const T &a, const T &b,
int idx) const>
struct SFINAE5 {};
826 template <
typename BinaryOpT,
bool (BinaryOpT::*)(const T &a, const T &b,
int idx)>
struct SFINAE6 {};
827 template <
typename BinaryOpT,
bool (BinaryOpT::*)(T a, T b,
int idx) const>
struct SFINAE7 {};
828 template <
typename BinaryOpT,
bool (BinaryOpT::*)(T a, T b,
int idx)>
struct SFINAE8 {};
830 template <
typename BinaryOpT>
static char Test(SFINAE1<BinaryOpT, &BinaryOpT::operator()> *);
831 template <
typename BinaryOpT>
static char Test(SFINAE2<BinaryOpT, &BinaryOpT::operator()> *);
832 template <
typename BinaryOpT>
static char Test(SFINAE3<BinaryOpT, &BinaryOpT::operator()> *);
833 template <
typename BinaryOpT>
static char Test(SFINAE4<BinaryOpT, &BinaryOpT::operator()> *);
835 template <
typename BinaryOpT>
static char Test(SFINAE5<BinaryOpT, &BinaryOpT::operator()> *);
836 template <
typename BinaryOpT>
static char Test(SFINAE6<BinaryOpT, &BinaryOpT::operator()> *);
837 template <
typename BinaryOpT>
static char Test(SFINAE7<BinaryOpT, &BinaryOpT::operator()> *);
838 template <
typename BinaryOpT>
static char Test(SFINAE8<BinaryOpT, &BinaryOpT::operator()> *);
840 template <
typename BinaryOpT>
static int Test(...);
845 static const bool HAS_PARAM =
sizeof(Test<BinaryOp>(NULL)) ==
sizeof(char);
848 #endif // DOXYGEN_SHOULD_SKIP_THIS
876 template <Category _CATEGORY,
bool _PRIMITIVE,
bool _NULL_TYPE,
typename _Un
signedBits>
883 PRIMITIVE = _PRIMITIVE,
884 NULL_TYPE = _NULL_TYPE,
888 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
893 template <
typename _Un
signedBits>
894 struct BaseTraits<UNSIGNED_INTEGER, true, false, _UnsignedBits>
896 typedef _UnsignedBits UnsignedBits;
898 static const Category CATEGORY = UNSIGNED_INTEGER;
899 static const UnsignedBits MIN_KEY = UnsignedBits(0);
900 static const UnsignedBits MAX_KEY = UnsignedBits(-1);
909 static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
914 static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
924 template <
typename _Un
signedBits>
925 struct BaseTraits<SIGNED_INTEGER, true, false, _UnsignedBits>
927 typedef _UnsignedBits UnsignedBits;
929 static const Category CATEGORY = SIGNED_INTEGER;
930 static const UnsignedBits HIGH_BIT = UnsignedBits(1) << ((
sizeof(UnsignedBits) * 8) - 1);
931 static const UnsignedBits MIN_KEY = HIGH_BIT;
932 static const UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT;
940 static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
942 return key ^ HIGH_BIT;
945 static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
947 return key ^ HIGH_BIT;
956 template <
typename _Un
signedBits>
957 struct BaseTraits<FLOATING_POINT, true, false, _UnsignedBits>
959 typedef _UnsignedBits UnsignedBits;
961 static const Category CATEGORY = FLOATING_POINT;
962 static const UnsignedBits HIGH_BIT = UnsignedBits(1) << ((
sizeof(UnsignedBits) * 8) - 1);
963 static const UnsignedBits MIN_KEY = UnsignedBits(-1);
964 static const UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT;
966 static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
968 UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT;
972 static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
974 UnsignedBits mask = (key & HIGH_BIT) ? HIGH_BIT : UnsignedBits(-1);
985 #endif // DOXYGEN_SHOULD_SKIP_THIS
993 #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document
997 template <>
struct NumericTraits<char> : BaseTraits<(std::numeric_limits<char>::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char> {};
998 template <>
struct NumericTraits<signed char> : BaseTraits<SIGNED_INTEGER, true, false, unsigned char> {};
999 template <>
struct NumericTraits<short> : BaseTraits<SIGNED_INTEGER, true, false, unsigned short> {};
1000 template <>
struct NumericTraits<int> : BaseTraits<SIGNED_INTEGER, true, false, unsigned int> {};
1001 template <>
struct NumericTraits<long> : BaseTraits<SIGNED_INTEGER, true, false, unsigned long> {};
1002 template <>
struct NumericTraits<long long> : BaseTraits<SIGNED_INTEGER, true, false, unsigned long long> {};
1004 template <>
struct NumericTraits<unsigned char> : BaseTraits<UNSIGNED_INTEGER, true, false, unsigned char> {};
1005 template <>
struct NumericTraits<unsigned short> : BaseTraits<UNSIGNED_INTEGER, true, false, unsigned short> {};
1006 template <>
struct NumericTraits<unsigned int> : BaseTraits<UNSIGNED_INTEGER, true, false, unsigned int> {};
1007 template <>
struct NumericTraits<unsigned long> : BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long> {};
1008 template <>
struct NumericTraits<unsigned long long> : BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long long> {};
1010 template <>
struct NumericTraits<float> : BaseTraits<FLOATING_POINT, true, false, unsigned int> {};
1011 template <>
struct NumericTraits<double> : BaseTraits<FLOATING_POINT, true, false, unsigned long long> {};
1013 #endif // DOXYGEN_SHOULD_SKIP_THIS
1019 template <
typename T>