2021-01-16 03:03:45 +02:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2020 Reimar Döffinger
|
2023-04-11 13:10:02 +02:00
|
|
|
* Copyright (c) 2023 xu fulong <839789740@qq.com>
|
2021-01-16 03:03:45 +02:00
|
|
|
*
|
|
|
|
|
* This file is part of FFmpeg.
|
|
|
|
|
*
|
|
|
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
*
|
|
|
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <stdint.h>
|
|
|
|
|
|
|
|
|
|
#include "libavutil/attributes.h"
|
|
|
|
|
#include "libavutil/cpu.h"
|
|
|
|
|
#include "libavutil/aarch64/cpu.h"
|
|
|
|
|
#include "libavcodec/hevcdsp.h"
|
|
|
|
|
|
2023-03-29 15:03:57 +02:00
|
|
|
void ff_hevc_v_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_v_loop_filter_chroma_10_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_v_loop_filter_chroma_12_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_chroma_10_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_chroma_12_neon(uint8_t *_pix, ptrdiff_t _stride,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
2024-02-13 02:09:28 +02:00
|
|
|
void ff_hevc_v_loop_filter_luma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_v_loop_filter_luma_10_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_v_loop_filter_luma_12_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_luma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_luma_10_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
|
|
|
|
void ff_hevc_h_loop_filter_luma_12_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
|
|
|
|
|
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_4x4_8_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_4x4_10_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-08-16 07:01:53 +02:00
|
|
|
void ff_hevc_add_residual_4x4_12_neon(uint8_t *_dst, const int16_t *coeffs,
|
|
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_8x8_8_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_8x8_10_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-08-16 07:01:53 +02:00
|
|
|
void ff_hevc_add_residual_8x8_12_neon(uint8_t *_dst, const int16_t *coeffs,
|
|
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_16x16_8_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_16x16_10_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-08-16 07:01:53 +02:00
|
|
|
void ff_hevc_add_residual_16x16_12_neon(uint8_t *_dst, const int16_t *coeffs,
|
|
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_32x32_8_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_add_residual_32x32_10_neon(uint8_t *_dst, const int16_t *coeffs,
|
2021-01-10 12:27:00 +02:00
|
|
|
ptrdiff_t stride);
|
2022-08-16 07:01:53 +02:00
|
|
|
void ff_hevc_add_residual_32x32_12_neon(uint8_t *_dst, const int16_t *coeffs,
|
|
|
|
|
ptrdiff_t stride);
|
2023-02-24 09:43:13 +02:00
|
|
|
void ff_hevc_idct_4x4_8_neon(int16_t *coeffs, int col_limit);
|
|
|
|
|
void ff_hevc_idct_4x4_10_neon(int16_t *coeffs, int col_limit);
|
2021-01-16 03:03:45 +02:00
|
|
|
void ff_hevc_idct_8x8_8_neon(int16_t *coeffs, int col_limit);
|
|
|
|
|
void ff_hevc_idct_8x8_10_neon(int16_t *coeffs, int col_limit);
|
|
|
|
|
void ff_hevc_idct_16x16_8_neon(int16_t *coeffs, int col_limit);
|
|
|
|
|
void ff_hevc_idct_16x16_10_neon(int16_t *coeffs, int col_limit);
|
2023-04-11 13:10:02 +02:00
|
|
|
void ff_hevc_idct_32x32_8_neon(int16_t *coeffs, int col_limit);
|
|
|
|
|
void ff_hevc_idct_32x32_10_neon(int16_t *coeffs, int col_limit);
|
2021-02-01 12:30:52 +02:00
|
|
|
void ff_hevc_idct_4x4_dc_8_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_8x8_dc_8_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_16x16_dc_8_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_32x32_dc_8_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_4x4_dc_10_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_8x8_dc_10_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_16x16_dc_10_neon(int16_t *coeffs);
|
|
|
|
|
void ff_hevc_idct_32x32_dc_10_neon(int16_t *coeffs);
|
2023-04-13 15:34:47 +02:00
|
|
|
void ff_hevc_transform_luma_4x4_neon_8(int16_t *coeffs);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_sao_band_filter_8x8_8_neon(uint8_t *_dst, const uint8_t *_src,
|
2021-01-07 13:55:44 +02:00
|
|
|
ptrdiff_t stride_dst, ptrdiff_t stride_src,
|
2022-07-22 06:10:59 +02:00
|
|
|
const int16_t *sao_offset_val, int sao_left_class,
|
2021-01-07 13:55:44 +02:00
|
|
|
int width, int height);
|
2022-07-22 06:10:59 +02:00
|
|
|
void ff_hevc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
|
|
|
|
|
const int16_t *sao_offset_val, int eo, int width, int height);
|
|
|
|
|
void ff_hevc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
|
|
|
|
|
const int16_t *sao_offset_val, int eo, int width, int height);
|
2022-10-11 09:09:02 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_h4_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_h6_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_h8_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_h12_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_h16_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_h32_8_neon(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
|
|
|
|
|
intptr_t mx, intptr_t my, int width);
|
2022-10-11 09:09:02 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_uni_h4_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,
|
|
|
|
|
int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_uni_h6_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,
|
|
|
|
|
int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_uni_h8_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,
|
|
|
|
|
int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_uni_h12_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t
|
|
|
|
|
my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_uni_h16_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t
|
|
|
|
|
my, int width);
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_uni_h32_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t
|
|
|
|
|
my, int width);
|
2022-10-11 09:09:02 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_bi_h4_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_bi_h6_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_bi_h8_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_bi_h12_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
|
|
|
|
void ff_hevc_put_hevc_qpel_bi_h16_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
void ff_hevc_put_hevc_qpel_bi_h32_8_neon(uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
|
|
|
|
|
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
|
|
|
|
|
mx, intptr_t my, int width);
|
2021-01-16 03:03:45 +02:00
|
|
|
|
2023-05-03 03:53:07 +02:00
|
|
|
#define NEON8_FNPROTO(fn, args, ext) \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##6_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##12_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##24_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##48_8_neon##ext args; \
|
2023-09-26 14:54:23 +02:00
|
|
|
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
|
2023-05-03 03:53:07 +02:00
|
|
|
|
|
|
|
|
#define NEON8_FNPROTO_PARTIAL_4(fn, args, ext) \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
|
2023-09-26 14:54:23 +02:00
|
|
|
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
|
2023-05-03 03:53:07 +02:00
|
|
|
|
2023-05-28 03:56:51 +02:00
|
|
|
#define NEON8_FNPROTO_PARTIAL_5(fn, args, ext) \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
|
|
|
|
|
void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
|
2023-09-26 14:54:23 +02:00
|
|
|
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
|
2023-05-28 03:56:51 +02:00
|
|
|
|
2023-10-26 03:11:09 +02:00
|
|
|
NEON8_FNPROTO(pel_pixels, (int16_t *dst,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-11-05 10:33:17 +02:00
|
|
|
NEON8_FNPROTO(pel_bi_pixels, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-11-11 11:54:35 +02:00
|
|
|
NEON8_FNPROTO(epel_bi_h, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-11-11 13:17:36 +02:00
|
|
|
NEON8_FNPROTO(epel_bi_v, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-11-11 13:57:40 +02:00
|
|
|
NEON8_FNPROTO(epel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-10-26 03:15:24 +02:00
|
|
|
NEON8_FNPROTO(epel_v, (int16_t *dst,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-05-07 10:58:30 +02:00
|
|
|
NEON8_FNPROTO(pel_uni_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
2023-05-05 16:06:22 +02:00
|
|
|
|
2023-05-03 03:53:07 +02:00
|
|
|
NEON8_FNPROTO(pel_uni_w_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-08-15 09:24:32 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_v, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-09-23 04:38:36 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_hv, (uint8_t *dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-05-27 03:42:07 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-05-03 03:53:07 +02:00
|
|
|
NEON8_FNPROTO_PARTIAL_4(qpel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2024-03-01 14:04:02 +02:00
|
|
|
NEON8_FNPROTO(epel_h, (int16_t *dst,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-05-28 04:30:28 +02:00
|
|
|
NEON8_FNPROTO(epel_h, (int16_t *dst,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-10-26 03:17:36 +02:00
|
|
|
NEON8_FNPROTO(epel_hv, (int16_t *dst,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2024-03-13 15:41:11 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-05-28 04:07:28 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width), _i8mm);
|
2023-05-28 03:56:51 +02:00
|
|
|
|
|
|
|
|
NEON8_FNPROTO(qpel_h, (int16_t *dst,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-10-26 03:24:32 +02:00
|
|
|
NEON8_FNPROTO(qpel_v, (int16_t *dst,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-10-26 03:33:35 +02:00
|
|
|
NEON8_FNPROTO(qpel_hv, (int16_t *dst,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-08-15 10:42:25 +02:00
|
|
|
NEON8_FNPROTO(qpel_uni_v, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-08-15 11:00:17 +02:00
|
|
|
NEON8_FNPROTO(qpel_uni_hv, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-05-05 16:06:22 +02:00
|
|
|
NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-05-28 04:35:43 +02:00
|
|
|
NEON8_FNPROTO(epel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-05-28 03:56:51 +02:00
|
|
|
NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
|
|
|
|
|
const uint8_t *_src, ptrdiff_t _srcstride,
|
|
|
|
|
int height, int denom, int wx, int ox,
|
|
|
|
|
intptr_t mx, intptr_t my, int width), _i8mm);
|
2023-05-05 16:06:22 +02:00
|
|
|
|
2023-11-12 02:32:10 +02:00
|
|
|
NEON8_FNPROTO(qpel_bi_v, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width),);
|
|
|
|
|
|
2023-11-12 03:03:28 +02:00
|
|
|
NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
|
|
|
|
|
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
|
|
|
|
|
int height, intptr_t mx, intptr_t my, int width), _i8mm);
|
|
|
|
|
|
2023-05-03 03:53:07 +02:00
|
|
|
#define NEON8_FNASSIGN(member, v, h, fn, ext) \
|
|
|
|
|
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
|
|
|
|
|
member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \
|
|
|
|
|
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
|
|
|
|
|
member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
|
|
|
|
|
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
|
|
|
|
|
member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
|
|
|
|
|
member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
|
|
|
|
|
member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon##ext; \
|
|
|
|
|
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
|
|
|
|
|
|
2024-03-01 14:04:02 +02:00
|
|
|
#define NEON8_FNASSIGN_SHARED_32(member, v, h, fn, ext) \
|
|
|
|
|
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
|
|
|
|
|
member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \
|
|
|
|
|
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
|
|
|
|
|
member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
|
|
|
|
|
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
|
|
|
|
|
member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
|
|
|
|
|
member[7][v][h] = \
|
|
|
|
|
member[8][v][h] = \
|
|
|
|
|
member[9][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext;
|
|
|
|
|
|
2023-05-03 03:53:07 +02:00
|
|
|
#define NEON8_FNASSIGN_PARTIAL_4(member, v, h, fn, ext) \
|
|
|
|
|
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
|
|
|
|
|
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
|
|
|
|
|
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
|
|
|
|
|
member[7][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
|
|
|
|
|
member[8][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
|
|
|
|
|
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
|
|
|
|
|
|
2023-05-28 03:56:51 +02:00
|
|
|
#define NEON8_FNASSIGN_PARTIAL_5(member, v, h, fn, ext) \
|
|
|
|
|
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
|
|
|
|
|
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
|
|
|
|
|
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
|
|
|
|
|
member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
|
|
|
|
|
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
|
2023-05-05 16:06:22 +02:00
|
|
|
|
2021-01-16 03:03:45 +02:00
|
|
|
av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth)
|
|
|
|
|
{
|
2023-05-05 16:06:22 +02:00
|
|
|
int cpu_flags = av_get_cpu_flags();
|
|
|
|
|
if (!have_neon(cpu_flags)) return;
|
2021-01-16 03:03:45 +02:00
|
|
|
|
|
|
|
|
if (bit_depth == 8) {
|
2024-02-13 02:09:28 +02:00
|
|
|
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_neon;
|
|
|
|
|
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_neon;
|
2023-03-29 15:03:57 +02:00
|
|
|
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_neon;
|
|
|
|
|
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_neon;
|
2021-01-10 12:27:00 +02:00
|
|
|
c->add_residual[0] = ff_hevc_add_residual_4x4_8_neon;
|
|
|
|
|
c->add_residual[1] = ff_hevc_add_residual_8x8_8_neon;
|
|
|
|
|
c->add_residual[2] = ff_hevc_add_residual_16x16_8_neon;
|
|
|
|
|
c->add_residual[3] = ff_hevc_add_residual_32x32_8_neon;
|
2023-02-24 09:43:13 +02:00
|
|
|
c->idct[0] = ff_hevc_idct_4x4_8_neon;
|
2021-01-16 03:03:45 +02:00
|
|
|
c->idct[1] = ff_hevc_idct_8x8_8_neon;
|
|
|
|
|
c->idct[2] = ff_hevc_idct_16x16_8_neon;
|
2023-04-11 13:10:02 +02:00
|
|
|
c->idct[3] = ff_hevc_idct_32x32_8_neon;
|
2021-02-01 12:30:52 +02:00
|
|
|
c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_neon;
|
|
|
|
|
c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_neon;
|
|
|
|
|
c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon;
|
|
|
|
|
c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon;
|
2023-04-13 15:34:47 +02:00
|
|
|
c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8;
|
2022-04-26 09:29:54 +02:00
|
|
|
c->sao_band_filter[0] =
|
|
|
|
|
c->sao_band_filter[1] =
|
|
|
|
|
c->sao_band_filter[2] =
|
|
|
|
|
c->sao_band_filter[3] =
|
|
|
|
|
c->sao_band_filter[4] = ff_hevc_sao_band_filter_8x8_8_neon;
|
2022-04-28 14:57:43 +02:00
|
|
|
c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8x8_8_neon;
|
2022-04-28 14:57:33 +02:00
|
|
|
c->sao_edge_filter[1] =
|
|
|
|
|
c->sao_edge_filter[2] =
|
|
|
|
|
c->sao_edge_filter[3] =
|
|
|
|
|
c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_16x16_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel[1][0][1] = ff_hevc_put_hevc_qpel_h4_8_neon;
|
|
|
|
|
c->put_hevc_qpel[2][0][1] = ff_hevc_put_hevc_qpel_h6_8_neon;
|
|
|
|
|
c->put_hevc_qpel[3][0][1] = ff_hevc_put_hevc_qpel_h8_8_neon;
|
|
|
|
|
c->put_hevc_qpel[4][0][1] =
|
|
|
|
|
c->put_hevc_qpel[6][0][1] = ff_hevc_put_hevc_qpel_h12_8_neon;
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel[5][0][1] = ff_hevc_put_hevc_qpel_h16_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel[7][0][1] =
|
|
|
|
|
c->put_hevc_qpel[8][0][1] =
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel[9][0][1] = ff_hevc_put_hevc_qpel_h32_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel_uni[1][0][1] = ff_hevc_put_hevc_qpel_uni_h4_8_neon;
|
|
|
|
|
c->put_hevc_qpel_uni[2][0][1] = ff_hevc_put_hevc_qpel_uni_h6_8_neon;
|
|
|
|
|
c->put_hevc_qpel_uni[3][0][1] = ff_hevc_put_hevc_qpel_uni_h8_8_neon;
|
|
|
|
|
c->put_hevc_qpel_uni[4][0][1] =
|
|
|
|
|
c->put_hevc_qpel_uni[6][0][1] = ff_hevc_put_hevc_qpel_uni_h12_8_neon;
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel_uni[5][0][1] = ff_hevc_put_hevc_qpel_uni_h16_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel_uni[7][0][1] =
|
|
|
|
|
c->put_hevc_qpel_uni[8][0][1] =
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel_uni[9][0][1] = ff_hevc_put_hevc_qpel_uni_h32_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel_bi[1][0][1] = ff_hevc_put_hevc_qpel_bi_h4_8_neon;
|
|
|
|
|
c->put_hevc_qpel_bi[2][0][1] = ff_hevc_put_hevc_qpel_bi_h6_8_neon;
|
|
|
|
|
c->put_hevc_qpel_bi[3][0][1] = ff_hevc_put_hevc_qpel_bi_h8_8_neon;
|
|
|
|
|
c->put_hevc_qpel_bi[4][0][1] =
|
|
|
|
|
c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_hevc_qpel_bi_h12_8_neon;
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_hevc_qpel_bi_h16_8_neon;
|
2022-10-11 09:09:02 +02:00
|
|
|
c->put_hevc_qpel_bi[7][0][1] =
|
|
|
|
|
c->put_hevc_qpel_bi[8][0][1] =
|
aarch64: hevc: Specialize put_hevc_\type\()_h*_8_neon for horizontal looping
For widths of 32 pixels and more, loop first horizontally,
then vertically.
Previously, this function would process a 16 pixel wide slice
of the block, looping vertically. After processing the whole
height, it would backtrack and process the next 16 pixel wide
slice.
When doing 8tap filtering horizontally, the function must load
7 more pixels (in practice, 8) following the actual inputs, and
this was done for each slice.
By iterating first horizontally throughout each line, then
vertically, we access data in a more cache friendly order, and
we don't need to reload data unnecessarily.
Keep the original order in put_hevc_\type\()_h12_8_neon; the
only suboptimal case there is for width=24. But specializing
an optimal variant for that would require more code, which
might not be worth it.
For the h16 case, this implementation would give a slowdown,
as it now loads the first 8 pixels separately from the rest, but
for larger widths, it is a gain. Therefore, keep the h16 case
as it was (but remove the outer loop), and create a new specialized
version for horizontal looping with 16 pixels at a time.
Before: Cortex A53 A72 A73 Graviton 3
put_hevc_qpel_h16_8_neon: 710.5 667.7 692.5 211.0
put_hevc_qpel_h32_8_neon: 2791.5 2643.5 2732.0 883.5
put_hevc_qpel_h64_8_neon: 10954.0 10657.0 10874.2 3241.5
After:
put_hevc_qpel_h16_8_neon: 697.5 663.5 705.7 212.5
put_hevc_qpel_h32_8_neon: 2767.2 2684.5 2791.2 920.5
put_hevc_qpel_h64_8_neon: 10559.2 10471.5 10932.2 3051.7
Signed-off-by: Martin Storsjö <martin@martin.st>
2024-03-24 12:54:13 +02:00
|
|
|
c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_hevc_qpel_bi_h32_8_neon;
|
2023-05-03 03:53:07 +02:00
|
|
|
|
2023-10-26 03:11:09 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel, 0, 0, pel_pixels,);
|
2023-10-26 03:15:24 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel, 1, 0, epel_v,);
|
2023-10-26 03:11:09 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel, 0, 0, pel_pixels,);
|
2023-10-26 03:24:32 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel, 1, 0, qpel_v,);
|
2023-11-05 10:33:17 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_bi, 0, 0, pel_bi_pixels,);
|
2023-11-11 11:54:35 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_bi, 0, 1, epel_bi_h,);
|
2023-11-11 13:17:36 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_bi, 1, 0, epel_bi_v,);
|
2023-11-05 10:33:17 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 0, 0, pel_bi_pixels,);
|
2023-11-12 02:32:10 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 0, qpel_bi_v,);
|
2023-05-07 10:58:30 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni, 0, 0, pel_uni_pixels,);
|
2023-08-15 09:24:32 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 0, epel_uni_v,);
|
2023-05-07 10:58:30 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 0, 0, pel_uni_pixels,);
|
2023-08-15 10:42:25 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 0, qpel_uni_v,);
|
2023-05-03 03:53:07 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 0, pel_uni_w_pixels,);
|
|
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 0, pel_uni_w_pixels,);
|
2023-05-27 03:42:07 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 0, epel_uni_w_v,);
|
2023-05-03 03:53:07 +02:00
|
|
|
NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
|
|
|
|
|
|
2024-03-01 14:04:02 +02:00
|
|
|
NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);
|
2024-03-13 15:41:11 +02:00
|
|
|
NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h,);
|
2024-03-01 14:04:02 +02:00
|
|
|
|
2023-05-05 16:06:22 +02:00
|
|
|
if (have_i8mm(cpu_flags)) {
|
2023-05-28 04:30:28 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);
|
2023-10-26 03:17:36 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);
|
2023-09-23 04:38:36 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 1, epel_uni_hv, _i8mm);
|
2023-05-28 04:07:28 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm);
|
2024-03-13 13:35:14 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv, _i8mm);
|
2023-11-11 13:57:40 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_epel_bi, 1, 1, epel_bi_hv, _i8mm);
|
2023-05-28 03:56:51 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm);
|
2023-10-26 03:33:35 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv, _i8mm);
|
2023-08-15 11:00:17 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 1, qpel_uni_hv, _i8mm);
|
2023-05-28 04:07:28 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm);
|
2023-05-28 03:56:51 +02:00
|
|
|
NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv, _i8mm);
|
2023-11-12 03:03:28 +02:00
|
|
|
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 1, qpel_bi_hv, _i8mm);
|
2023-05-05 16:06:22 +02:00
|
|
|
}
|
|
|
|
|
|
2021-01-16 03:03:45 +02:00
|
|
|
}
|
|
|
|
|
if (bit_depth == 10) {
|
2024-02-13 02:09:28 +02:00
|
|
|
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_neon;
|
|
|
|
|
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_neon;
|
2023-03-29 15:03:57 +02:00
|
|
|
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_neon;
|
|
|
|
|
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_neon;
|
2021-01-10 12:27:00 +02:00
|
|
|
c->add_residual[0] = ff_hevc_add_residual_4x4_10_neon;
|
|
|
|
|
c->add_residual[1] = ff_hevc_add_residual_8x8_10_neon;
|
|
|
|
|
c->add_residual[2] = ff_hevc_add_residual_16x16_10_neon;
|
|
|
|
|
c->add_residual[3] = ff_hevc_add_residual_32x32_10_neon;
|
2023-02-24 09:43:13 +02:00
|
|
|
c->idct[0] = ff_hevc_idct_4x4_10_neon;
|
2021-01-16 03:03:45 +02:00
|
|
|
c->idct[1] = ff_hevc_idct_8x8_10_neon;
|
|
|
|
|
c->idct[2] = ff_hevc_idct_16x16_10_neon;
|
2023-04-11 13:10:02 +02:00
|
|
|
c->idct[3] = ff_hevc_idct_32x32_10_neon;
|
2021-02-01 12:30:52 +02:00
|
|
|
c->idct_dc[0] = ff_hevc_idct_4x4_dc_10_neon;
|
|
|
|
|
c->idct_dc[1] = ff_hevc_idct_8x8_dc_10_neon;
|
|
|
|
|
c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_neon;
|
|
|
|
|
c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_neon;
|
2021-01-16 03:03:45 +02:00
|
|
|
}
|
2022-08-16 07:01:53 +02:00
|
|
|
if (bit_depth == 12) {
|
2024-02-13 02:09:28 +02:00
|
|
|
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_neon;
|
|
|
|
|
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_neon;
|
2023-03-29 15:03:57 +02:00
|
|
|
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_neon;
|
|
|
|
|
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_neon;
|
2022-08-16 07:01:53 +02:00
|
|
|
c->add_residual[0] = ff_hevc_add_residual_4x4_12_neon;
|
|
|
|
|
c->add_residual[1] = ff_hevc_add_residual_8x8_12_neon;
|
|
|
|
|
c->add_residual[2] = ff_hevc_add_residual_16x16_12_neon;
|
|
|
|
|
c->add_residual[3] = ff_hevc_add_residual_32x32_12_neon;
|
|
|
|
|
}
|
2021-01-16 03:03:45 +02:00
|
|
|
}
|
