Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2020, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
static inline void copy_128(const uint8_t *src, uint8_t *dst) {
__m128i s[8];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 16));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 16));
s[2] = _mm_loadu_si128((__m128i *)(src + 2 * 16));
s[3] = _mm_loadu_si128((__m128i *)(src + 3 * 16));
s[4] = _mm_loadu_si128((__m128i *)(src + 4 * 16));
s[5] = _mm_loadu_si128((__m128i *)(src + 5 * 16));
s[6] = _mm_loadu_si128((__m128i *)(src + 6 * 16));
s[7] = _mm_loadu_si128((__m128i *)(src + 7 * 16));
_mm_store_si128((__m128i *)(dst + 0 * 16), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 16), s[1]);
_mm_store_si128((__m128i *)(dst + 2 * 16), s[2]);
_mm_store_si128((__m128i *)(dst + 3 * 16), s[3]);
_mm_store_si128((__m128i *)(dst + 4 * 16), s[4]);
_mm_store_si128((__m128i *)(dst + 5 * 16), s[5]);
_mm_store_si128((__m128i *)(dst + 6 * 16), s[6]);
_mm_store_si128((__m128i *)(dst + 7 * 16), s[7]);
}
void aom_convolve_copy_sse2(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride, int w, int h) {
// The w >= 16 cases use _mm_store_si128(), which requires its output address
// be aligned on a 16-byte boundary.
if (w >= 16) {
assert(!((intptr_t)dst % 16));
assert(!(dst_stride % 16));
}
if (w == 2) {
do {
memmove(dst, src, 2 * sizeof(*src));
src += src_stride;
dst += dst_stride;
memmove(dst, src, 2 * sizeof(*src));
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 4) {
do {
memmove(dst, src, 4 * sizeof(*src));
src += src_stride;
dst += dst_stride;
memmove(dst, src, 4 * sizeof(*src));
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 8) {
do {
__m128i s[2];
s[0] = _mm_loadl_epi64((__m128i *)src);
src += src_stride;
s[1] = _mm_loadl_epi64((__m128i *)src);
src += src_stride;
_mm_storel_epi64((__m128i *)dst, s[0]);
dst += dst_stride;
_mm_storel_epi64((__m128i *)dst, s[1]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 16) {
do {
__m128i s[2];
s[0] = _mm_loadu_si128((__m128i *)src);
src += src_stride;
s[1] = _mm_loadu_si128((__m128i *)src);
src += src_stride;
_mm_store_si128((__m128i *)dst, s[0]);
dst += dst_stride;
_mm_store_si128((__m128i *)dst, s[1]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 32) {
do {
__m128i s[4];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 16));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 16));
src += src_stride;
s[2] = _mm_loadu_si128((__m128i *)(src + 0 * 16));
s[3] = _mm_loadu_si128((__m128i *)(src + 1 * 16));
src += src_stride;
_mm_store_si128((__m128i *)(dst + 0 * 16), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 16), s[1]);
dst += dst_stride;
_mm_store_si128((__m128i *)(dst + 0 * 16), s[2]);
_mm_store_si128((__m128i *)(dst + 1 * 16), s[3]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 64) {
do {
__m128i s[8];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 16));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 16));
s[2] = _mm_loadu_si128((__m128i *)(src + 2 * 16));
s[3] = _mm_loadu_si128((__m128i *)(src + 3 * 16));
src += src_stride;
s[4] = _mm_loadu_si128((__m128i *)(src + 0 * 16));
s[5] = _mm_loadu_si128((__m128i *)(src + 1 * 16));
s[6] = _mm_loadu_si128((__m128i *)(src + 2 * 16));
s[7] = _mm_loadu_si128((__m128i *)(src + 3 * 16));
src += src_stride;
_mm_store_si128((__m128i *)(dst + 0 * 16), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 16), s[1]);
_mm_store_si128((__m128i *)(dst + 2 * 16), s[2]);
_mm_store_si128((__m128i *)(dst + 3 * 16), s[3]);
dst += dst_stride;
_mm_store_si128((__m128i *)(dst + 0 * 16), s[4]);
_mm_store_si128((__m128i *)(dst + 1 * 16), s[5]);
_mm_store_si128((__m128i *)(dst + 2 * 16), s[6]);
_mm_store_si128((__m128i *)(dst + 3 * 16), s[7]);
dst += dst_stride;
h -= 2;
} while (h);
} else {
do {
copy_128(src, dst);
src += src_stride;
dst += dst_stride;
copy_128(src, dst);
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
}
}
#if CONFIG_AV1_HIGHBITDEPTH
static inline void highbd_copy_64(const uint16_t *src, uint16_t *dst) {
__m128i s[8];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
s[2] = _mm_loadu_si128((__m128i *)(src + 2 * 8));
s[3] = _mm_loadu_si128((__m128i *)(src + 3 * 8));
s[4] = _mm_loadu_si128((__m128i *)(src + 4 * 8));
s[5] = _mm_loadu_si128((__m128i *)(src + 5 * 8));
s[6] = _mm_loadu_si128((__m128i *)(src + 6 * 8));
s[7] = _mm_loadu_si128((__m128i *)(src + 7 * 8));
_mm_store_si128((__m128i *)(dst + 0 * 8), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[1]);
_mm_store_si128((__m128i *)(dst + 2 * 8), s[2]);
_mm_store_si128((__m128i *)(dst + 3 * 8), s[3]);
_mm_store_si128((__m128i *)(dst + 4 * 8), s[4]);
_mm_store_si128((__m128i *)(dst + 5 * 8), s[5]);
_mm_store_si128((__m128i *)(dst + 6 * 8), s[6]);
_mm_store_si128((__m128i *)(dst + 7 * 8), s[7]);
}
static inline void highbd_copy_128(const uint16_t *src, uint16_t *dst) {
__m128i s[16];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
s[2] = _mm_loadu_si128((__m128i *)(src + 2 * 8));
s[3] = _mm_loadu_si128((__m128i *)(src + 3 * 8));
s[4] = _mm_loadu_si128((__m128i *)(src + 4 * 8));
s[5] = _mm_loadu_si128((__m128i *)(src + 5 * 8));
s[6] = _mm_loadu_si128((__m128i *)(src + 6 * 8));
s[7] = _mm_loadu_si128((__m128i *)(src + 7 * 8));
s[8] = _mm_loadu_si128((__m128i *)(src + 8 * 8));
s[9] = _mm_loadu_si128((__m128i *)(src + 9 * 8));
s[10] = _mm_loadu_si128((__m128i *)(src + 10 * 8));
s[11] = _mm_loadu_si128((__m128i *)(src + 11 * 8));
s[12] = _mm_loadu_si128((__m128i *)(src + 12 * 8));
s[13] = _mm_loadu_si128((__m128i *)(src + 13 * 8));
s[14] = _mm_loadu_si128((__m128i *)(src + 14 * 8));
s[15] = _mm_loadu_si128((__m128i *)(src + 15 * 8));
_mm_store_si128((__m128i *)(dst + 0 * 8), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[1]);
_mm_store_si128((__m128i *)(dst + 2 * 8), s[2]);
_mm_store_si128((__m128i *)(dst + 3 * 8), s[3]);
_mm_store_si128((__m128i *)(dst + 4 * 8), s[4]);
_mm_store_si128((__m128i *)(dst + 5 * 8), s[5]);
_mm_store_si128((__m128i *)(dst + 6 * 8), s[6]);
_mm_store_si128((__m128i *)(dst + 7 * 8), s[7]);
_mm_store_si128((__m128i *)(dst + 8 * 8), s[8]);
_mm_store_si128((__m128i *)(dst + 9 * 8), s[9]);
_mm_store_si128((__m128i *)(dst + 10 * 8), s[10]);
_mm_store_si128((__m128i *)(dst + 11 * 8), s[11]);
_mm_store_si128((__m128i *)(dst + 12 * 8), s[12]);
_mm_store_si128((__m128i *)(dst + 13 * 8), s[13]);
_mm_store_si128((__m128i *)(dst + 14 * 8), s[14]);
_mm_store_si128((__m128i *)(dst + 15 * 8), s[15]);
}
void aom_highbd_convolve_copy_sse2(const uint16_t *src, ptrdiff_t src_stride,
uint16_t *dst, ptrdiff_t dst_stride, int w,
int h) {
// The w >= 8 cases use _mm_store_si128(), which requires its output address
// be aligned on a 16-byte boundary.
if (w >= 8) {
assert(!((intptr_t)dst % 16));
assert(!(dst_stride % 8));
}
if (w == 2) {
do {
__m128i s = _mm_loadl_epi64((__m128i *)src);
*(int *)dst = _mm_cvtsi128_si32(s);
src += src_stride;
dst += dst_stride;
s = _mm_loadl_epi64((__m128i *)src);
*(int *)dst = _mm_cvtsi128_si32(s);
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 4) {
do {
__m128i s[2];
s[0] = _mm_loadl_epi64((__m128i *)src);
src += src_stride;
s[1] = _mm_loadl_epi64((__m128i *)src);
src += src_stride;
_mm_storel_epi64((__m128i *)dst, s[0]);
dst += dst_stride;
_mm_storel_epi64((__m128i *)dst, s[1]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 8) {
do {
__m128i s[2];
s[0] = _mm_loadu_si128((__m128i *)src);
src += src_stride;
s[1] = _mm_loadu_si128((__m128i *)src);
src += src_stride;
_mm_store_si128((__m128i *)dst, s[0]);
dst += dst_stride;
_mm_store_si128((__m128i *)dst, s[1]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 16) {
do {
__m128i s[4];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
src += src_stride;
s[2] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[3] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
src += src_stride;
_mm_store_si128((__m128i *)(dst + 0 * 8), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[1]);
dst += dst_stride;
_mm_store_si128((__m128i *)(dst + 0 * 8), s[2]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[3]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 32) {
do {
__m128i s[8];
s[0] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[1] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
s[2] = _mm_loadu_si128((__m128i *)(src + 2 * 8));
s[3] = _mm_loadu_si128((__m128i *)(src + 3 * 8));
src += src_stride;
s[4] = _mm_loadu_si128((__m128i *)(src + 0 * 8));
s[5] = _mm_loadu_si128((__m128i *)(src + 1 * 8));
s[6] = _mm_loadu_si128((__m128i *)(src + 2 * 8));
s[7] = _mm_loadu_si128((__m128i *)(src + 3 * 8));
src += src_stride;
_mm_store_si128((__m128i *)(dst + 0 * 8), s[0]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[1]);
_mm_store_si128((__m128i *)(dst + 2 * 8), s[2]);
_mm_store_si128((__m128i *)(dst + 3 * 8), s[3]);
dst += dst_stride;
_mm_store_si128((__m128i *)(dst + 0 * 8), s[4]);
_mm_store_si128((__m128i *)(dst + 1 * 8), s[5]);
_mm_store_si128((__m128i *)(dst + 2 * 8), s[6]);
_mm_store_si128((__m128i *)(dst + 3 * 8), s[7]);
dst += dst_stride;
h -= 2;
} while (h);
} else if (w == 64) {
do {
highbd_copy_64(src, dst);
src += src_stride;
dst += dst_stride;
highbd_copy_64(src, dst);
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
} else {
do {
highbd_copy_128(src, dst);
src += src_stride;
dst += dst_stride;
highbd_copy_128(src, dst);
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h);
}
}
#endif // CONFIG_AV1_HIGHBITDEPTH