/* * By downloading, copying, installing or using the software you agree to this license. * If you do not agree to this license, do not download, install, * copy or use the software. * * * License Agreement * For Open Source Computer Vision Library * (3-clause BSD License) * * Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. * Third party copyrights are property of their respective owners. * * Redistribution and use in source and binary forms, with or without modification, * are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * * Neither the names of the copyright holders nor the names of the contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * This software is provided by the copyright holders and contributors "as is" and * any express or implied warranties, including, but not limited to, the implied * warranties of merchantability and fitness for a particular purpose are disclaimed. * In no event shall copyright holders or contributors be liable for any direct, * indirect, incidental, special, exemplary, or consequential damages * (including, but not limited to, procurement of substitute goods or services; * loss of use, data, or profits; or business interruption) however caused * and on any theory of liability, whether in contract, strict liability, * or tort (including negligence or otherwise) arising in any way out of * the use of this software, even if advised of the possibility of such damage. */ #include "common.hpp" #include "vtransform.hpp" namespace CAROTENE_NS { void extract2(const Size2D &size, const u8 * srcBase, ptrdiff_t srcStride, u8 * dstBase, ptrdiff_t dstStride, u32 coi) { internal::assertSupportedConfiguration(); #ifdef CAROTENE_NEON #ifndef __ANDROID__ size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; #endif size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; for (size_t i = 0u; i < size.height; ++i) { const u8 * src = internal::getRowPtr(srcBase, srcStride, i); u8 * dst = internal::getRowPtr(dstBase, dstStride, i); size_t sj = 0u, dj = 0u; #ifndef __ANDROID__ for (; dj < roiw32; sj += 64, dj += 32) { internal::prefetch(src + sj); uint8x16x2_t v_src = vld2q_u8(src + sj); vst1q_u8(dst + dj, v_src.val[coi]); v_src = vld2q_u8(src + sj + 32); vst1q_u8(dst + dj + 16, v_src.val[coi]); } #endif for (; dj < roiw8; sj += 16, dj += 8) { uint8x8x2_t v_src = vld2_u8(src + sj); vst1_u8(dst + dj, v_src.val[coi]); } for (; dj < size.width; sj += 2, ++dj) { dst[dj] = src[sj + coi]; } } #else (void)size; (void)srcBase; (void)srcStride; (void)dstBase; (void)dstStride; (void)coi; #endif } void extract3(const Size2D &size, const u8 * srcBase, ptrdiff_t srcStride, u8 * dstBase, ptrdiff_t dstStride, u32 coi) { internal::assertSupportedConfiguration(); #ifdef CAROTENE_NEON #ifndef __ANDROID__ size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; #endif size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; for (size_t i = 0u; i < size.height; ++i) { const u8 * src = internal::getRowPtr(srcBase, srcStride, i); u8 * dst = internal::getRowPtr(dstBase, dstStride, i); size_t sj = 0u, dj = 0u; #ifndef __ANDROID__ for (; dj < roiw32; sj += 96, dj += 32) { internal::prefetch(src + sj); uint8x16x3_t v_src = vld3q_u8(src + sj); vst1q_u8(dst + dj, v_src.val[coi]); v_src = vld3q_u8(src + sj + 48); vst1q_u8(dst + dj + 16, v_src.val[coi]); } #endif for (; dj < roiw8; sj += 24, dj += 8) { uint8x8x3_t v_src = vld3_u8(src + sj); vst1_u8(dst + dj, v_src.val[coi]); } for (; dj < size.width; sj += 3, ++dj) { dst[dj] = src[sj + coi]; } } #else (void)size; (void)srcBase; (void)srcStride; (void)dstBase; (void)dstStride; (void)coi; #endif } void extract4(const Size2D &size, const u8 * srcBase, ptrdiff_t srcStride, u8 * dstBase, ptrdiff_t dstStride, u32 coi) { internal::assertSupportedConfiguration(); #ifdef CAROTENE_NEON #ifndef __ANDROID__ size_t roiw32 = size.width >= 31 ? size.width - 31 : 0; #endif size_t roiw8 = size.width >= 7 ? size.width - 7 : 0; for (size_t i = 0u; i < size.height; ++i) { const u8 * src = internal::getRowPtr(srcBase, srcStride, i); u8 * dst = internal::getRowPtr(dstBase, dstStride, i); size_t sj = 0u, dj = 0u; #ifndef __ANDROID__ for (; dj < roiw32; sj += 128, dj += 32) { internal::prefetch(src + sj); uint8x16x4_t v_src = vld4q_u8(src + sj); vst1q_u8(dst + dj, v_src.val[coi]); v_src = vld4q_u8(src + sj + 64); vst1q_u8(dst + dj + 16, v_src.val[coi]); } #endif for (; dj < roiw8; sj += 32, dj += 8) { uint8x8x4_t v_src = vld4_u8(src + sj); vst1_u8(dst + dj, v_src.val[coi]); } for (; dj < size.width; sj += 4, ++dj) { dst[dj] = src[sj + coi]; } } #else (void)size; (void)srcBase; (void)srcStride; (void)dstBase; (void)dstStride; (void)coi; #endif } #define FILL_LINES2(macro,type) \ macro##_LINE(type,0) \ macro##_LINE(type,1) #define FILL_LINES3(macro,type) \ FILL_LINES2(macro,type) \ macro##_LINE(type,2) #define FILL_LINES4(macro,type) \ FILL_LINES3(macro,type) \ macro##_LINE(type,3) #define FARG_LINE(type, n) , type * dst##n##Base, ptrdiff_t dst##n##Stride #ifdef CAROTENE_NEON #define VROW_LINE(type, n) type * dst##n = internal::getRowPtr(dst##n##Base, dst##n##Stride, i); #define VST1Q_LINE(type, n) vst1q_##type(dst##n + dj, v_src.val[n]); #define VST1_LINE(type, n) vst1_##type(dst##n + dj, v_src.val[n]); #define SST_LINE(type, n) dst##n[dj] = src[sj + n]; #define MUL2(val) (val << 1) #define MUL3(val) (MUL2(val) + val) #define MUL4(val) (val << 2) #define CONTDST2 srcStride == dst0Stride && \ srcStride == dst1Stride && #define CONTDST3 srcStride == dst0Stride && \ srcStride == dst1Stride && \ srcStride == dst2Stride && #define CONTDST4 srcStride == dst0Stride && \ srcStride == dst1Stride && \ srcStride == dst2Stride && \ srcStride == dst3Stride && #if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__) #define SPLIT_ASM2(sgn, bits) __asm__ ( \ "vld2." #bits " {d0, d2}, [%[in0]] \n\t" \ "vld2." #bits " {d1, d3}, [%[in1]] \n\t" \ "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ : \ : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), \ [in0] "r" (src + sj), [in1] "r" (src + sj + MUL2(8)/sizeof(sgn##bits)) \ : "d0","d1","d2","d3" \ ); #define SPLIT_ASM3(sgn, bits) __asm__ ( \ "vld3." #bits " {d0, d2, d4}, [%[in0]] \n\t" \ "vld3." #bits " {d1, d3, d5}, [%[in1]] \n\t" \ "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ "vst1." #bits " {d4-d5}, [%[out2]] \n\t" \ : \ : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), \ [in0] "r" (src + sj), [in1] "r" (src + sj + MUL3(8)/sizeof(sgn##bits)) \ : "d0","d1","d2","d3","d4","d5" \ ); #define SPLIT_ASM4(sgn, bits) __asm__ ( \ "vld4." #bits " {d0, d2, d4, d6}, [%[in0]] \n\t" \ "vld4." #bits " {d1, d3, d5, d7}, [%[in1]] \n\t" \ "vst1." #bits " {d0-d1}, [%[out0]] \n\t" \ "vst1." #bits " {d2-d3}, [%[out1]] \n\t" \ "vst1." #bits " {d4-d5}, [%[out2]] \n\t" \ "vst1." #bits " {d6-d7}, [%[out3]] \n\t" \ : \ : [out0] "r" (dst0 + dj), [out1] "r" (dst1 + dj), [out2] "r" (dst2 + dj), [out3] "r" (dst3 + dj), \ [in0] "r" (src + sj), [in1] "r" (src + sj + MUL4(8)/sizeof(sgn##bits)) \ : "d0","d1","d2","d3","d4","d5","d6","d7" \ ); #define SPLIT_QUAD(sgn, bits, n) { \ internal::prefetch(src + sj); \ SPLIT_ASM##n(sgn, bits) \ } #else #define SPLIT_QUAD(sgn, bits, n) { \ internal::prefetch(src + sj); \ vec128 v_src = vld##n##q_##sgn##bits(src + sj); \ FILL_LINES##n(VST1Q, sgn##bits) \ } #endif #define SPLIT(sgn,bits,n) void split##n(const Size2D &_size, \ const sgn##bits * srcBase, ptrdiff_t srcStride \ FILL_LINES##n(FARG, sgn##bits) ) \ { \ internal::assertSupportedConfiguration(); \ Size2D size(_size); \ if (CONTDST##n \ dst0Stride == (ptrdiff_t)(size.width)) \ { \ size.width *= size.height; \ size.height = 1; \ } \ typedef internal::VecTraits::vec128 vec128; \ size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \ typedef internal::VecTraits::vec64 vec64; \ size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0; \ \ for (size_t i = 0u; i < size.height; ++i) \ { \ const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i); \ FILL_LINES##n(VROW, sgn##bits) \ size_t sj = 0u, dj = 0u; \ \ for (; dj < roiw16; sj += MUL##n(16)/sizeof(sgn##bits), dj += 16/sizeof(sgn##bits)) \ SPLIT_QUAD(sgn, bits, n) \ \ if (dj < roiw8) \ { \ vec64 v_src = vld##n##_##sgn##bits(src + sj); \ FILL_LINES##n(VST1, sgn##bits) \ sj += MUL##n(8)/sizeof(sgn##bits); \ dj += 8/sizeof(sgn##bits); \ } \ \ for (; dj < size.width; sj += n, ++dj) \ { \ FILL_LINES##n(SST, sgn##bits) \ } \ } \ } #define SPLIT64(sgn,n) void split##n(const Size2D &_size, \ const sgn##64 * srcBase, ptrdiff_t srcStride \ FILL_LINES##n(FARG, sgn##64) ) \ { \ internal::assertSupportedConfiguration(); \ Size2D size(_size); \ if (CONTDST##n \ dst0Stride == (ptrdiff_t)(size.width)) \ { \ size.width *= size.height; \ size.height = 1; \ } \ typedef internal::VecTraits::vec64 vec64; \ \ for (size_t i = 0u; i < size.height; ++i) \ { \ const sgn##64 * src = internal::getRowPtr(srcBase, srcStride, i); \ FILL_LINES##n(VROW, sgn##64) \ size_t sj = 0u, dj = 0u; \ \ for (; dj < size.width; sj += n, ++dj) \ { \ vec64 v_src = vld##n##_##sgn##64(src + sj); \ FILL_LINES##n(VST1, sgn##64) \ } \ } \ } #if !defined(__aarch64__) && defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7 && !defined(__clang__) #define ALPHA_QUAD(sgn, bits) { \ internal::prefetch(src + sj); \ __asm__ ( \ "vld4." #bits " {d0, d2, d4, d6}, [%[in0]] \n\t" \ "vld4." #bits " {d1, d3, d5, d7}, [%[in1]] \n\t" \ "vst3." #bits " {d0, d2, d4}, [%[out3_1]] \n\t" \ "vst3." #bits " {d1, d3, d5}, [%[out3_2]] \n\t" \ "vst1." #bits " {d6-d7}, [%[out1]] \n\t" \ : \ : [out3_1] "r" (dst3 + d3j), [out3_2] "r" (dst3 + d3j + 24/sizeof(sgn##bits)), [out1] "r" (dst1 + d1j), \ [in0] "r" (src + sj), [in1] "r" (src + sj + 32/sizeof(sgn##bits)) \ : "d0","d1","d2","d3","d4","d5","d6","d7" \ ); \ } #else #define ALPHA_QUAD(sgn, bits) { \ internal::prefetch(src + sj); \ union { vec128_4 v4; vec128_3 v3; } vals; \ vals.v4 = vld4q_##sgn##bits(src + sj); \ vst3q_##sgn##bits(dst3 + d3j, vals.v3); \ vst1q_##sgn##bits(dst1 + d1j, vals.v4.val[3]); \ } #endif #define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &_size, \ const sgn##bits * srcBase, ptrdiff_t srcStride, \ sgn##bits * dst3Base, ptrdiff_t dst3Stride, \ sgn##bits * dst1Base, ptrdiff_t dst1Stride) \ { \ internal::assertSupportedConfiguration(); \ Size2D size(_size); \ if (srcStride == dst3Stride && \ srcStride == dst1Stride && \ srcStride == (ptrdiff_t)(size.width)) \ { \ size.width *= size.height; \ size.height = 1; \ } \ typedef internal::VecTraits::vec128 vec128_4; \ typedef internal::VecTraits::vec128 vec128_3; \ size_t roiw16 = size.width >= (16/sizeof(sgn##bits)-1) ? size.width - (16/sizeof(sgn##bits)-1) : 0; \ typedef internal::VecTraits::vec64 vec64_4; \ typedef internal::VecTraits::vec64 vec64_3; \ size_t roiw8 = size.width >= (8/sizeof(sgn##bits)-1) ? size.width - (8/sizeof(sgn##bits)-1) : 0; \ \ for (size_t i = 0u; i < size.height; ++i) \ { \ const sgn##bits * src = internal::getRowPtr(srcBase, srcStride, i); \ sgn##bits * dst3 = internal::getRowPtr(dst3Base, dst3Stride, i); \ sgn##bits * dst1 = internal::getRowPtr(dst1Base, dst1Stride, i); \ size_t sj = 0u, d3j = 0u, d1j = 0u; \ \ for (; d1j < roiw16; sj += MUL4(16)/sizeof(sgn##bits), d3j += MUL3(16)/sizeof(sgn##bits), \ d1j += 16/sizeof(sgn##bits)) \ ALPHA_QUAD(sgn, bits) \ \ if (d1j < roiw8) \ { \ union { vec64_4 v4; vec64_3 v3; } vals; \ vals.v4 = vld4_##sgn##bits(src + sj); \ vst3_u8(dst3 + d3j, vals.v3); \ vst1_u8(dst1 + d1j, vals.v4.val[3]); \ sj += MUL4(8)/sizeof(sgn##bits); \ d3j += MUL3(8)/sizeof(sgn##bits); \ d1j += 8/sizeof(sgn##bits); \ } \ \ for (; d1j < size.width; sj += 4, d3j += 3, ++d1j) \ { \ dst3[d3j+0] = src[sj + 0]; \ dst3[d3j+1] = src[sj + 1]; \ dst3[d3j+2] = src[sj + 2]; \ dst1[d1j] = src[sj + 3]; \ } \ } \ } #else #define VOID_LINE(type, n) (void)dst##n##Base; (void)dst##n##Stride; #define SPLIT(sgn,bits,n) void split##n(const Size2D &size, \ const sgn##bits * srcBase, ptrdiff_t srcStride \ FILL_LINES##n(FARG, sgn##bits) ) \ { \ internal::assertSupportedConfiguration(); \ (void)size; \ (void)srcBase; \ (void)srcStride; \ FILL_LINES##n(VOID, sgn##bits) \ } #define SPLIT64(sgn,n) SPLIT(sgn,64,n) #define SPLIT4ALPHA(sgn,bits) void split4(const Size2D &size, \ const sgn##bits * srcBase, ptrdiff_t srcStride, \ sgn##bits * dst3Base, ptrdiff_t dst3Stride, \ sgn##bits * dst1Base, ptrdiff_t dst1Stride) \ { \ internal::assertSupportedConfiguration(); \ (void)size; \ (void)srcBase; \ (void)srcStride; \ (void)dst3Base; \ (void)dst3Stride; \ (void)dst1Base; \ (void)dst1Stride; \ } #endif //CAROTENE_NEON SPLIT(u, 8,2) SPLIT(u, 8,3) SPLIT(u, 8,4) SPLIT(u,16,2) SPLIT(u,16,3) SPLIT(u,16,4) SPLIT(s,32,2) SPLIT(s,32,3) SPLIT(s,32,4) SPLIT64(s, 2) SPLIT64(s, 3) SPLIT64(s, 4) SPLIT4ALPHA(u,8) } // namespace CAROTENE_NS