You've already forked FFmpeg
mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2025-11-23 21:54:53 +02:00
982d3a98d0
This covers most 8-bit and 16-bit ops, and some 32-bit ops. It also covers all floating point operations. While this is not yet 100% coverage, it's good enough for the vast majority of formats out there. Of special note is the packed shuffle fast path, which uses pshufb at vector sizes up to AVX512.
1063 lines
28 KiB
NASM
1063 lines
28 KiB
NASM
;******************************************************************************
|
|
;* Copyright (c) 2025 Niklas Haas
|
|
;*
|
|
;* This file is part of FFmpeg.
|
|
;*
|
|
;* FFmpeg is free software; you can redistribute it and/or
|
|
;* modify it under the terms of the GNU Lesser General Public
|
|
;* License as published by the Free Software Foundation; either
|
|
;* version 2.1 of the License, or (at your option) any later version.
|
|
;*
|
|
;* FFmpeg is distributed in the hope that it will be useful,
|
|
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
;* Lesser General Public License for more details.
|
|
;*
|
|
;* You should have received a copy of the GNU Lesser General Public
|
|
;* License along with FFmpeg; if not, write to the Free Software
|
|
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
;******************************************************************************
|
|
|
|
%include "ops_common.asm"
|
|
|
|
SECTION_RODATA
|
|
|
|
align 16
|
|
expand16_shuf: db 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14
|
|
expand32_shuf: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12
|
|
|
|
read8_unpack2: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15
|
|
read8_unpack3: db 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, -1, -1, -1, -1
|
|
read8_unpack4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15
|
|
read16_unpack2: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
|
|
read16_unpack3: db 0, 1, 6, 7, 2, 3, 8, 9, 4, 5, 10, 11, -1, -1, -1, -1
|
|
read16_unpack4: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
|
|
write8_pack2: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
|
|
write8_pack3: db 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, -1, -1, -1, -1
|
|
write16_pack3: db 0, 1, 4, 5, 8, 9, 2, 3, 6, 7, 10, 11, -1, -1, -1, -1
|
|
|
|
%define write8_pack4 read8_unpack4
|
|
%define write16_pack4 read16_unpack2
|
|
%define write16_pack2 read16_unpack4
|
|
|
|
align 32
|
|
bits_shuf: db 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, \
|
|
2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3
|
|
bits_mask: db 128, 64, 32, 16, 8, 4, 2, 1,128, 64, 32, 16, 8, 4, 2, 1
|
|
bits_reverse: db 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8,
|
|
|
|
align 32
|
|
mask1: times 32 db 0x01
|
|
mask2: times 32 db 0x03
|
|
mask3: times 32 db 0x07
|
|
mask4: times 32 db 0x0F
|
|
|
|
SECTION .text
|
|
|
|
;---------------------------------------------------------
|
|
; Global entry point. See `ops_common.asm` for info.
|
|
|
|
%macro process_fn 1 ; number of planes
|
|
cglobal sws_process%1_x86, 6, 6 + 2 * %1, 16
|
|
; Args:
|
|
; execq, implq, bxd, yd as defined in ops_common.int
|
|
; bx_end and y_end are initially in tmp0d / tmp1d
|
|
; (see SwsOpFunc signature)
|
|
;
|
|
; Stack layout:
|
|
; [rsp + 0] = [qword] impl->cont (address of first kernel)
|
|
; [rsp + 8] = [qword] &impl[1] (restore implq after chain)
|
|
; [rsp + 16] = [dword] bx start (restore after line finish)
|
|
; [rsp + 20] = [dword] bx end (loop counter limit)
|
|
; [rsp + 24] = [dword] y end (loop counter limit)
|
|
sub rsp, 32
|
|
mov [rsp + 16], bxd
|
|
mov [rsp + 20], tmp0d ; bx_end
|
|
mov [rsp + 24], tmp1d ; y_end
|
|
mov tmp0q, [implq + SwsOpImpl.cont]
|
|
add implq, SwsOpImpl.next
|
|
mov [rsp + 0], tmp0q
|
|
mov [rsp + 8], implq
|
|
|
|
; load plane pointers
|
|
mov in0q, [execq + SwsOpExec.in0]
|
|
IF %1 > 1, mov in1q, [execq + SwsOpExec.in1]
|
|
IF %1 > 2, mov in2q, [execq + SwsOpExec.in2]
|
|
IF %1 > 3, mov in3q, [execq + SwsOpExec.in3]
|
|
mov out0q, [execq + SwsOpExec.out0]
|
|
IF %1 > 1, mov out1q, [execq + SwsOpExec.out1]
|
|
IF %1 > 2, mov out2q, [execq + SwsOpExec.out2]
|
|
IF %1 > 3, mov out3q, [execq + SwsOpExec.out3]
|
|
jmp [rsp] ; call into op chain
|
|
|
|
; Declare a separate global label for the return point, so that we can append
|
|
; it to the list of op function pointers from the C code, effectively ensuring
|
|
; that we end up here again after the op chain finishes processing a line.
|
|
; (See also: cglobal_label in x86inc.asm)
|
|
%if FORMAT_ELF
|
|
global current_function %+ _return:function hidden
|
|
%elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
|
|
global current_function %+ _return:private_extern
|
|
%else
|
|
global current_function %+ _return
|
|
%endif
|
|
align function_align
|
|
current_function %+ _return:
|
|
|
|
; op chain always returns back here
|
|
mov implq, [rsp + 8]
|
|
inc bxd
|
|
cmp bxd, [rsp + 20]
|
|
jne .continue
|
|
; end of line
|
|
inc yd
|
|
cmp yd, [rsp + 24]
|
|
je .end
|
|
; bump addresses to point to start of next line
|
|
add in0q, [execq + SwsOpExec.in_bump0]
|
|
IF %1 > 1, add in1q, [execq + SwsOpExec.in_bump1]
|
|
IF %1 > 2, add in2q, [execq + SwsOpExec.in_bump2]
|
|
IF %1 > 3, add in3q, [execq + SwsOpExec.in_bump3]
|
|
add out0q, [execq + SwsOpExec.out_bump0]
|
|
IF %1 > 1, add out1q, [execq + SwsOpExec.out_bump1]
|
|
IF %1 > 2, add out2q, [execq + SwsOpExec.out_bump2]
|
|
IF %1 > 3, add out3q, [execq + SwsOpExec.out_bump3]
|
|
mov bxd, [rsp + 16]
|
|
.continue:
|
|
jmp [rsp]
|
|
.end:
|
|
add rsp, 32
|
|
RET
|
|
%endmacro
|
|
|
|
process_fn 1
|
|
process_fn 2
|
|
process_fn 3
|
|
process_fn 4
|
|
|
|
;---------------------------------------------------------
|
|
; Packed shuffle fast-path
|
|
|
|
; This is a special entry point for handling a subset of operation chains
|
|
; that can be reduced down to a single `pshufb` shuffle mask. For more details
|
|
; about when this works, refer to the documentation of `ff_sws_solve_shuffle`.
|
|
;
|
|
; We specialize this function for every possible combination of pixel strides.
|
|
; For example, gray -> gray16 is classified as an "8, 16" operation because it
|
|
; takes 8 bytes and expands them out to 16 bytes in each application of the
|
|
; 128-bit shuffle mask.
|
|
;
|
|
; Since pshufb can't shuffle across lanes, we only instantiate SSE4 versions for
|
|
; all shuffles that are not a clean multiple of 128 bits (e.g. rgb24 -> rgb0).
|
|
; For the clean multiples (e.g. rgba -> argb), we also define AVX2 and AVX512
|
|
; versions that can handle a larger number of bytes at once.
|
|
|
|
%macro packed_shuffle 2 ; size_in, size_out
|
|
cglobal packed_shuffle%1_%2, 6, 10, 2, \
|
|
exec, shuffle, bx, y, bxend, yend, src, dst, src_stride, dst_stride
|
|
mov srcq, [execq + SwsOpExec.in0]
|
|
mov dstq, [execq + SwsOpExec.out0]
|
|
mov src_strideq, [execq + SwsOpExec.in_stride0]
|
|
mov dst_strideq, [execq + SwsOpExec.out_stride0]
|
|
VBROADCASTI128 m1, [shuffleq]
|
|
sub bxendd, bxd
|
|
sub yendd, yd
|
|
; reuse now-unneeded regs
|
|
%define srcidxq execq
|
|
imul srcidxq, bxendq, -%1
|
|
%if %1 = %2
|
|
%define dstidxq srcidxq
|
|
%else
|
|
%define dstidxq shuffleq ; no longer needed reg
|
|
imul dstidxq, bxendq, -%2
|
|
%endif
|
|
sub srcq, srcidxq
|
|
sub dstq, dstidxq
|
|
.loop:
|
|
%if %1 <= 4
|
|
movd m0, [srcq + srcidxq]
|
|
%elif %1 <= 8
|
|
movq m0, [srcq + srcidxq]
|
|
%else
|
|
movu m0, [srcq + srcidxq]
|
|
%endif
|
|
pshufb m0, m1
|
|
movu [dstq + dstidxq], m0
|
|
add srcidxq, %1
|
|
IF %1 != %2,add dstidxq, %2
|
|
jnz .loop
|
|
add srcq, src_strideq
|
|
add dstq, dst_strideq
|
|
imul srcidxq, bxendq, -%1
|
|
IF %1 != %2,imul dstidxq, bxendq, -%2
|
|
dec yendd
|
|
jnz .loop
|
|
RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse4
|
|
packed_shuffle 5, 15 ; 8 -> 24
|
|
packed_shuffle 4, 16 ; 8 -> 32, 16 -> 64
|
|
packed_shuffle 2, 12 ; 8 -> 48
|
|
packed_shuffle 16, 8 ; 16 -> 8
|
|
packed_shuffle 10, 15 ; 16 -> 24
|
|
packed_shuffle 8, 16 ; 16 -> 32, 32 -> 64
|
|
packed_shuffle 4, 12 ; 16 -> 48
|
|
packed_shuffle 15, 15 ; 24 -> 24
|
|
packed_shuffle 12, 16 ; 24 -> 32
|
|
packed_shuffle 6, 12 ; 24 -> 48
|
|
packed_shuffle 16, 12 ; 32 -> 24, 64 -> 48
|
|
packed_shuffle 16, 16 ; 32 -> 32, 64 -> 64
|
|
packed_shuffle 8, 12 ; 32 -> 48
|
|
packed_shuffle 12, 12 ; 48 -> 48
|
|
|
|
INIT_YMM avx2
|
|
packed_shuffle 32, 32
|
|
|
|
INIT_ZMM avx512
|
|
packed_shuffle 64, 64
|
|
|
|
;---------------------------------------------------------
|
|
; Planar reads / writes
|
|
|
|
%macro read_planar 1 ; elems
|
|
op read_planar%1
|
|
movu mx, [in0q]
|
|
IF %1 > 1, movu my, [in1q]
|
|
IF %1 > 2, movu mz, [in2q]
|
|
IF %1 > 3, movu mw, [in3q]
|
|
%if V2
|
|
movu mx2, [in0q + mmsize]
|
|
IF %1 > 1, movu my2, [in1q + mmsize]
|
|
IF %1 > 2, movu mz2, [in2q + mmsize]
|
|
IF %1 > 3, movu mw2, [in3q + mmsize]
|
|
%endif
|
|
LOAD_CONT tmp0q
|
|
add in0q, mmsize * (1 + V2)
|
|
IF %1 > 1, add in1q, mmsize * (1 + V2)
|
|
IF %1 > 2, add in2q, mmsize * (1 + V2)
|
|
IF %1 > 3, add in3q, mmsize * (1 + V2)
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro write_planar 1 ; elems
|
|
op write_planar%1
|
|
LOAD_CONT tmp0q
|
|
movu [out0q], mx
|
|
IF %1 > 1, movu [out1q], my
|
|
IF %1 > 2, movu [out2q], mz
|
|
IF %1 > 3, movu [out3q], mw
|
|
%if V2
|
|
movu [out0q + mmsize], mx2
|
|
IF %1 > 1, movu [out1q + mmsize], my2
|
|
IF %1 > 2, movu [out2q + mmsize], mz2
|
|
IF %1 > 3, movu [out3q + mmsize], mw2
|
|
%endif
|
|
add out0q, mmsize * (1 + V2)
|
|
IF %1 > 1, add out1q, mmsize * (1 + V2)
|
|
IF %1 > 2, add out2q, mmsize * (1 + V2)
|
|
IF %1 > 3, add out3q, mmsize * (1 + V2)
|
|
FINISH tmp0q
|
|
%endmacro
|
|
|
|
%macro read_packed2 1 ; depth
|
|
op read%1_packed2
|
|
movu m8, [in0q + 0*mmsize]
|
|
movu m9, [in0q + 1*mmsize]
|
|
IF V2, movu m10, [in0q + 2*mmsize]
|
|
IF V2, movu m11, [in0q + 3*mmsize]
|
|
IF %1 < 32, VBROADCASTI128 m12, [read%1_unpack2]
|
|
LOAD_CONT tmp0q
|
|
add in0q, mmsize * (2 + V2 * 2)
|
|
%if %1 == 32
|
|
shufps m8, m8, q3120
|
|
shufps m9, m9, q3120
|
|
IF V2, shufps m10, m10, q3120
|
|
IF V2, shufps m11, m11, q3120
|
|
%else
|
|
pshufb m8, m12 ; { X0 Y0 | X1 Y1 }
|
|
pshufb m9, m12 ; { X2 Y2 | X3 Y3 }
|
|
IF V2, pshufb m10, m12
|
|
IF V2, pshufb m11, m12
|
|
%endif
|
|
unpcklpd mx, m8, m9 ; { X0 X2 | X1 X3 }
|
|
unpckhpd my, m8, m9 ; { Y0 Y2 | Y1 Y3 }
|
|
IF V2, unpcklpd mx2, m10, m11
|
|
IF V2, unpckhpd my2, m10, m11
|
|
%if avx_enabled
|
|
vpermq mx, mx, q3120 ; { X0 X1 | X2 X3 }
|
|
vpermq my, my, q3120 ; { Y0 Y1 | Y2 Y3 }
|
|
IF V2, vpermq mx2, mx2, q3120
|
|
IF V2, vpermq my2, my2, q3120
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro write_packed2 1 ; depth
|
|
op write%1_packed2
|
|
IF %1 < 32, VBROADCASTI128 m12, [write%1_pack2]
|
|
LOAD_CONT tmp0q
|
|
%if avx_enabled
|
|
vpermq mx, mx, q3120 ; { X0 X2 | X1 X3 }
|
|
vpermq my, my, q3120 ; { Y0 Y2 | Y1 Y3 }
|
|
IF V2, vpermq mx2, mx2, q3120
|
|
IF V2, vpermq my2, my2, q3120
|
|
%endif
|
|
unpcklpd m8, mx, my ; { X0 Y0 | X1 Y1 }
|
|
unpckhpd m9, mx, my ; { X2 Y2 | X3 Y3 }
|
|
IF V2, unpcklpd m10, mx2, my2
|
|
IF V2, unpckhpd m11, mx2, my2
|
|
%if %1 == 32
|
|
shufps m8, m8, q3120
|
|
shufps m9, m9, q3120
|
|
IF V2, shufps m10, m10, q3120
|
|
IF V2, shufps m11, m11, q3120
|
|
%else
|
|
pshufb m8, m12
|
|
pshufb m9, m12
|
|
IF V2, pshufb m10, m12
|
|
IF V2, pshufb m11, m12
|
|
%endif
|
|
movu [out0q + 0*mmsize], m8
|
|
movu [out0q + 1*mmsize], m9
|
|
IF V2, movu [out0q + 2*mmsize], m10
|
|
IF V2, movu [out0q + 3*mmsize], m11
|
|
add out0q, mmsize * (2 + V2 * 2)
|
|
FINISH tmp0q
|
|
%endmacro
|
|
|
|
; helper macro reused for both 3 and 4 component packed reads
|
|
%macro read_packed_inner 7 ; x, y, z, w, addr, num, depth
|
|
movu xm8, [%5 + 0 * %6]
|
|
movu xm9, [%5 + 4 * %6]
|
|
movu xm10, [%5 + 8 * %6]
|
|
movu xm11, [%5 + 12 * %6]
|
|
%if avx_enabled
|
|
vinserti128 m8, m8, [%5 + 16 * %6], 1
|
|
vinserti128 m9, m9, [%5 + 20 * %6], 1
|
|
vinserti128 m10, m10, [%5 + 24 * %6], 1
|
|
vinserti128 m11, m11, [%5 + 28 * %6], 1
|
|
%endif
|
|
%if %7 == 32
|
|
mova %1, m8
|
|
mova %2, m9
|
|
mova %3, m10
|
|
mova %4, m11
|
|
%else
|
|
pshufb %1, m8, m12 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
|
|
pshufb %2, m9, m12 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
|
|
pshufb %3, m10, m12 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
|
|
pshufb %4, m11, m12 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
|
|
%endif
|
|
punpckldq m8, %1, %2 ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 }
|
|
punpckldq m9, %3, %4 ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 }
|
|
punpckhdq m10, %1, %2 ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 }
|
|
punpckhdq m11, %3, %4 ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 }
|
|
punpcklqdq %1, m8, m9 ; { X0 X1 X2 X3 | X4 X5 X6 X7 }
|
|
punpckhqdq %2, m8, m9 ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 }
|
|
punpcklqdq %3, m10, m11 ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 }
|
|
IF %6 > 3, punpckhqdq %4, m10, m11 ; { W0 W1 W2 W3 | W4 W5 W6 W7 }
|
|
%endmacro
|
|
|
|
%macro read_packed 2 ; num, depth
|
|
op read%2_packed%1
|
|
IF %2 < 32, VBROADCASTI128 m12, [read%2_unpack%1]
|
|
LOAD_CONT tmp0q
|
|
read_packed_inner mx, my, mz, mw, in0q, %1, %2
|
|
IF1 V2, read_packed_inner mx2, my2, mz2, mw2, in0q + %1 * mmsize, %1, %2
|
|
add in0q, %1 * mmsize * (1 + V2)
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro write_packed_inner 7 ; x, y, z, w, addr, num, depth
|
|
punpckldq m8, %1, %2 ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 }
|
|
punpckldq m9, %3, %4 ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 }
|
|
punpckhdq m10, %1, %2 ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 }
|
|
punpckhdq m11, %3, %4 ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 }
|
|
punpcklqdq %1, m8, m9 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
|
|
punpckhqdq %2, m8, m9 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
|
|
punpcklqdq %3, m10, m11 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
|
|
punpckhqdq %4, m10, m11 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
|
|
%if %7 == 32
|
|
mova m8, %1
|
|
mova m9, %2
|
|
mova m10, %3
|
|
mova m11, %4
|
|
%else
|
|
pshufb m8, %1, m12
|
|
pshufb m9, %2, m12
|
|
pshufb m10, %3, m12
|
|
pshufb m11, %4, m12
|
|
%endif
|
|
movu [%5 + 0*%6], xm8
|
|
movu [%5 + 4*%6], xm9
|
|
movu [%5 + 8*%6], xm10
|
|
movu [%5 + 12*%6], xm11
|
|
%if avx_enabled
|
|
vextracti128 [%5 + 16*%6], m8, 1
|
|
vextracti128 [%5 + 20*%6], m9, 1
|
|
vextracti128 [%5 + 24*%6], m10, 1
|
|
vextracti128 [%5 + 28*%6], m11, 1
|
|
%endif
|
|
%endmacro
|
|
|
|
%macro write_packed 2 ; num, depth
|
|
op write%2_packed%1
|
|
IF %2 < 32, VBROADCASTI128 m12, [write%2_pack%1]
|
|
LOAD_CONT tmp0q
|
|
write_packed_inner mx, my, mz, mw, out0q, %1, %2
|
|
IF1 V2, write_packed_inner mx2, my2, mz2, mw2, out0q + %1 * mmsize, %1, %2
|
|
add out0q, %1 * mmsize * (1 + V2)
|
|
FINISH tmp0q
|
|
%endmacro
|
|
|
|
%macro rw_packed 1 ; depth
|
|
read_packed2 %1
|
|
read_packed 3, %1
|
|
read_packed 4, %1
|
|
write_packed2 %1
|
|
write_packed 3, %1
|
|
write_packed 4, %1
|
|
%endmacro
|
|
|
|
%macro read_nibbles 0
|
|
op read_nibbles1
|
|
%if avx_enabled
|
|
movu xmx, [in0q]
|
|
IF V2, movu xmx2, [in0q + 16]
|
|
%else
|
|
movq xmx, [in0q]
|
|
IF V2, movq xmx2, [in0q + 8]
|
|
%endif
|
|
VBROADCASTI128 m8, [mask4]
|
|
LOAD_CONT tmp0q
|
|
add in0q, (mmsize >> 1) * (1 + V2)
|
|
pmovzxbw mx, xmx
|
|
IF V2, pmovzxbw mx2, xmx2
|
|
psllw my, mx, 8
|
|
IF V2, psllw my2, mx2, 8
|
|
psrlw mx, 4
|
|
IF V2, psrlw mx2, 4
|
|
pand my, m8
|
|
IF V2, pand my2, m8
|
|
por mx, my
|
|
IF V2, por mx2, my2
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro read_bits 0
|
|
op read_bits1
|
|
%if avx_enabled
|
|
vpbroadcastd mx, [in0q]
|
|
IF V2, vpbroadcastd mx2, [in0q + 4]
|
|
%else
|
|
movd mx, [in0q]
|
|
IF V2, movd mx2, [in0q + 2]
|
|
%endif
|
|
mova m8, [bits_shuf]
|
|
VBROADCASTI128 m9, [bits_mask]
|
|
VBROADCASTI128 m10, [mask1]
|
|
LOAD_CONT tmp0q
|
|
add in0q, (mmsize >> 3) * (1 + V2)
|
|
pshufb mx, m8
|
|
IF V2, pshufb mx2, m8
|
|
pand mx, m9
|
|
IF V2, pand mx2, m9
|
|
pcmpeqb mx, m9
|
|
IF V2, pcmpeqb mx2, m9
|
|
pand mx, m10
|
|
IF V2, pand mx2, m10
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
; TODO: write_nibbles
|
|
|
|
%macro write_bits 0
|
|
op write_bits1
|
|
VBROADCASTI128 m8, [bits_reverse]
|
|
psllw mx, 7
|
|
IF V2, psllw mx2, 7
|
|
pshufb mx, m8
|
|
IF V2, pshufb mx2, m8
|
|
pmovmskb tmp0d, mx
|
|
IF V2, pmovmskb tmp1d, mx2
|
|
%if avx_enabled
|
|
mov [out0q], tmp0d
|
|
IF V2, mov [out0q + 4], tmp1d
|
|
%else
|
|
mov [out0q], tmp0d
|
|
IF V2, mov [out0q + 2], tmp1d
|
|
%endif
|
|
LOAD_CONT tmp0q
|
|
add out0q, (mmsize >> 3) * (1 + V2)
|
|
FINISH tmp0q
|
|
%endmacro
|
|
|
|
;--------------------------
|
|
; Pixel packing / unpacking
|
|
|
|
%macro pack_generic 3-4 0 ; x, y, z, w
|
|
op pack_%1%2%3%4
|
|
; pslld works for all sizes because the input should not overflow
|
|
IF %2, pslld mx, %4+%3+%2
|
|
IF %3, pslld my, %4+%3
|
|
IF %4, pslld mz, %4
|
|
IF %2, por mx, my
|
|
IF %3, por mx, mz
|
|
IF %4, por mx, mw
|
|
%if V2
|
|
IF %2, pslld mx2, %4+%3+%2
|
|
IF %3, pslld my2, %4+%3
|
|
IF %4, pslld mz2, %4
|
|
IF %2, por mx2, my2
|
|
IF %3, por mx2, mz2
|
|
IF %4, por mx2, mw2
|
|
%endif
|
|
CONTINUE
|
|
%endmacro
|
|
|
|
%macro unpack 5-6 0 ; type, bits, x, y, z, w
|
|
op unpack_%3%4%5%6
|
|
; clear high bits by shifting left
|
|
IF %6, vpsll%1 mw, mx, %2 - (%6)
|
|
IF %5, vpsll%1 mz, mx, %2 - (%6+%5)
|
|
IF %4, vpsll%1 my, mx, %2 - (%6+%5+%4)
|
|
psrl%1 mx, %4+%5+%6
|
|
IF %4, psrl%1 my, %2 - %4
|
|
IF %5, psrl%1 mz, %2 - %5
|
|
IF %6, psrl%1 mw, %2 - %6
|
|
%if V2
|
|
IF %6, vpsll%1 mw2, mx2, %2 - (%6)
|
|
IF %5, vpsll%1 mz2, mx2, %2 - (%6+%5)
|
|
IF %4, vpsll%1 my2, mx2, %2 - (%6+%5+%4)
|
|
psrl%1 mx2, %4+%5+%6
|
|
IF %4, psrl%1 my2, %2 - %4
|
|
IF %5, psrl%1 mz2, %2 - %5
|
|
IF %6, psrl%1 mw2, %2 - %6
|
|
%endif
|
|
CONTINUE
|
|
%endmacro
|
|
|
|
%macro unpack8 3 ; x, y, z
|
|
op unpack_%1%2%3 %+ 0
|
|
pand mz, mx, [mask%3]
|
|
psrld my, mx, %3
|
|
psrld mx, %3+%2
|
|
pand my, [mask%2]
|
|
pand mx, [mask%1]
|
|
%if V2
|
|
pand mz2, mx2, [mask%3]
|
|
psrld my2, mx2, %3
|
|
psrld mx2, %3+%2
|
|
pand my2, [mask%2]
|
|
pand mx2, [mask%1]
|
|
%endif
|
|
CONTINUE
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Generic byte order shuffle (packed swizzle, endian, etc)
|
|
|
|
%macro shuffle 0
|
|
op shuffle
|
|
VBROADCASTI128 m8, [implq + SwsOpImpl.priv]
|
|
LOAD_CONT tmp0q
|
|
IF X, pshufb mx, m8
|
|
IF Y, pshufb my, m8
|
|
IF Z, pshufb mz, m8
|
|
IF W, pshufb mw, m8
|
|
%if V2
|
|
IF X, pshufb mx2, m8
|
|
IF Y, pshufb my2, m8
|
|
IF Z, pshufb mz2, m8
|
|
IF W, pshufb mw2, m8
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Clearing
|
|
|
|
%macro clear_alpha 3 ; idx, vreg, vreg2
|
|
op clear_alpha%1
|
|
LOAD_CONT tmp0q
|
|
pcmpeqb %2, %2
|
|
IF V2, mova %3, %2
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro clear_zero 3 ; idx, vreg, vreg2
|
|
op clear_zero%1
|
|
LOAD_CONT tmp0q
|
|
pxor %2, %2
|
|
IF V2, mova %3, %2
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
; note: the pattern is inverted for these functions; i.e. X=1 implies that we
|
|
; *keep* the X component, not that we clear it
|
|
%macro clear_generic 0
|
|
op clear
|
|
LOAD_CONT tmp0q
|
|
%if avx_enabled
|
|
IF !X, vpbroadcastd mx, [implq + SwsOpImpl.priv + 0]
|
|
IF !Y, vpbroadcastd my, [implq + SwsOpImpl.priv + 4]
|
|
IF !Z, vpbroadcastd mz, [implq + SwsOpImpl.priv + 8]
|
|
IF !W, vpbroadcastd mw, [implq + SwsOpImpl.priv + 12]
|
|
%else ; !avx_enabled
|
|
IF !X, movd mx, [implq + SwsOpImpl.priv + 0]
|
|
IF !Y, movd my, [implq + SwsOpImpl.priv + 4]
|
|
IF !Z, movd mz, [implq + SwsOpImpl.priv + 8]
|
|
IF !W, movd mw, [implq + SwsOpImpl.priv + 12]
|
|
IF !X, pshufd mx, mx, 0
|
|
IF !Y, pshufd my, my, 0
|
|
IF !Z, pshufd mz, mz, 0
|
|
IF !W, pshufd mw, mw, 0
|
|
%endif
|
|
%if V2
|
|
IF !X, mova mx2, mx
|
|
IF !Y, mova my2, my
|
|
IF !Z, mova mz2, mz
|
|
IF !W, mova mw2, mw
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro clear_funcs 0
|
|
decl_pattern 1, 1, 1, 0, clear_generic
|
|
decl_pattern 0, 1, 1, 1, clear_generic
|
|
decl_pattern 0, 0, 1, 1, clear_generic
|
|
decl_pattern 1, 0, 0, 1, clear_generic
|
|
decl_pattern 1, 1, 0, 0, clear_generic
|
|
decl_pattern 0, 1, 0, 1, clear_generic
|
|
decl_pattern 1, 0, 1, 0, clear_generic
|
|
decl_pattern 1, 0, 0, 0, clear_generic
|
|
decl_pattern 0, 1, 0, 0, clear_generic
|
|
decl_pattern 0, 0, 1, 0, clear_generic
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Swizzling and duplicating
|
|
|
|
; mA := mB, mB := mC, ... mX := mA across both halves
|
|
%macro vrotate 2-* ; A, B, C, ...
|
|
%rep %0
|
|
%assign rot_a %1 + 4
|
|
%assign rot_b %2 + 4
|
|
mova m%1, m%2
|
|
IF V2, mova m%[rot_a], m%[rot_b]
|
|
%rotate 1
|
|
%endrep
|
|
%undef rot_a
|
|
%undef rot_b
|
|
%endmacro
|
|
|
|
%macro swizzle_funcs 0
|
|
op swizzle_3012
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3, 2, 1
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3021
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3, 1
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_2103
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3210
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3
|
|
vrotate 8, 1, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3102
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3201
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3, 1, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_1203
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 1, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_1023
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 1
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_2013
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 2, 1
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_2310
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 2, 1, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_2130
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 2, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_1230
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 1, 2, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_1320
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 1, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0213
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 1, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0231
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 1, 2, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0312
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 1, 3, 2
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3120
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 0, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0321
|
|
LOAD_CONT tmp0q
|
|
vrotate 8, 1, 3
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0003
|
|
LOAD_CONT tmp0q
|
|
mova my, mx
|
|
mova mz, mx
|
|
%if V2
|
|
mova my2, mx2
|
|
mova mz2, mx2
|
|
%endif
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_0001
|
|
LOAD_CONT tmp0q
|
|
mova mw, my
|
|
mova mz, mx
|
|
mova my, mx
|
|
%if V2
|
|
mova mw2, my2
|
|
mova mz2, mx2
|
|
mova my2, mx2
|
|
%endif
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_3000
|
|
LOAD_CONT tmp0q
|
|
mova my, mx
|
|
mova mz, mx
|
|
mova mx, mw
|
|
mova mw, my
|
|
%if V2
|
|
mova my2, mx2
|
|
mova mz2, mx2
|
|
mova mx2, mw2
|
|
mova mw2, my2
|
|
%endif
|
|
CONTINUE tmp0q
|
|
|
|
op swizzle_1000
|
|
LOAD_CONT tmp0q
|
|
mova mz, mx
|
|
mova mw, mx
|
|
mova mx, my
|
|
mova my, mz
|
|
%if V2
|
|
mova mz2, mx2
|
|
mova mw2, mx2
|
|
mova mx2, my2
|
|
mova my2, mz2
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Pixel type conversions
|
|
|
|
%macro conv8to16 1 ; type
|
|
op %1_U8_U16
|
|
LOAD_CONT tmp0q
|
|
%if V2
|
|
%if avx_enabled
|
|
IF X, vextracti128 xmx2, mx, 1
|
|
IF Y, vextracti128 xmy2, my, 1
|
|
IF Z, vextracti128 xmz2, mz, 1
|
|
IF W, vextracti128 xmw2, mw, 1
|
|
%else
|
|
IF X, psrldq xmx2, mx, 8
|
|
IF Y, psrldq xmy2, my, 8
|
|
IF Z, psrldq xmz2, mz, 8
|
|
IF W, psrldq xmw2, mw, 8
|
|
%endif
|
|
IF X, pmovzxbw mx2, xmx2
|
|
IF Y, pmovzxbw my2, xmy2
|
|
IF Z, pmovzxbw mz2, xmz2
|
|
IF W, pmovzxbw mw2, xmw2
|
|
%endif ; V2
|
|
IF X, pmovzxbw mx, xmx
|
|
IF Y, pmovzxbw my, xmy
|
|
IF Z, pmovzxbw mz, xmz
|
|
IF W, pmovzxbw mw, xmw
|
|
|
|
%ifidn %1, expand
|
|
VBROADCASTI128 m8, [expand16_shuf]
|
|
%if V2
|
|
IF X, pshufb mx2, m8
|
|
IF Y, pshufb my2, m8
|
|
IF Z, pshufb mz2, m8
|
|
IF W, pshufb mw2, m8
|
|
%endif
|
|
IF X, pshufb mx, m8
|
|
IF Y, pshufb my, m8
|
|
IF Z, pshufb mz, m8
|
|
IF W, pshufb mw, m8
|
|
%endif ; expand
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro conv16to8 0
|
|
op convert_U16_U8
|
|
LOAD_CONT tmp0q
|
|
%if V2
|
|
; this code technically works for the !V2 case as well, but slower
|
|
IF X, packuswb mx, mx2
|
|
IF Y, packuswb my, my2
|
|
IF Z, packuswb mz, mz2
|
|
IF W, packuswb mw, mw2
|
|
IF X, vpermq mx, mx, q3120
|
|
IF Y, vpermq my, my, q3120
|
|
IF Z, vpermq mz, mz, q3120
|
|
IF W, vpermq mw, mw, q3120
|
|
%else
|
|
IF X, vextracti128 xm8, mx, 1
|
|
IF Y, vextracti128 xm9, my, 1
|
|
IF Z, vextracti128 xm10, mz, 1
|
|
IF W, vextracti128 xm11, mw, 1
|
|
vzeroupper
|
|
IF X, packuswb xmx, xm8
|
|
IF Y, packuswb xmy, xm9
|
|
IF Z, packuswb xmz, xm10
|
|
IF W, packuswb xmw, xm11
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro conv8to32 1 ; type
|
|
op %1_U8_U32
|
|
LOAD_CONT tmp0q
|
|
IF X, psrldq xmx2, xmx, 8
|
|
IF Y, psrldq xmy2, xmy, 8
|
|
IF Z, psrldq xmz2, xmz, 8
|
|
IF W, psrldq xmw2, xmw, 8
|
|
IF X, pmovzxbd mx, xmx
|
|
IF Y, pmovzxbd my, xmy
|
|
IF Z, pmovzxbd mz, xmz
|
|
IF W, pmovzxbd mw, xmw
|
|
IF X, pmovzxbd mx2, xmx2
|
|
IF Y, pmovzxbd my2, xmy2
|
|
IF Z, pmovzxbd mz2, xmz2
|
|
IF W, pmovzxbd mw2, xmw2
|
|
%ifidn %1, expand
|
|
VBROADCASTI128 m8, [expand32_shuf]
|
|
IF X, pshufb mx, m8
|
|
IF Y, pshufb my, m8
|
|
IF Z, pshufb mz, m8
|
|
IF W, pshufb mw, m8
|
|
IF X, pshufb mx2, m8
|
|
IF Y, pshufb my2, m8
|
|
IF Z, pshufb mz2, m8
|
|
IF W, pshufb mw2, m8
|
|
%endif ; expand
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro conv32to8 0
|
|
op convert_U32_U8
|
|
LOAD_CONT tmp0q
|
|
IF X, packusdw mx, mx2
|
|
IF Y, packusdw my, my2
|
|
IF Z, packusdw mz, mz2
|
|
IF W, packusdw mw, mw2
|
|
IF X, vextracti128 xmx2, mx, 1
|
|
IF Y, vextracti128 xmy2, my, 1
|
|
IF Z, vextracti128 xmz2, mz, 1
|
|
IF W, vextracti128 xmw2, mw, 1
|
|
vzeroupper
|
|
IF X, packuswb xmx, xmx2
|
|
IF Y, packuswb xmy, xmy2
|
|
IF Z, packuswb xmz, xmz2
|
|
IF W, packuswb xmw, xmw2
|
|
IF X, vpshufd xmx, xmx, q3120
|
|
IF Y, vpshufd xmy, xmy, q3120
|
|
IF Z, vpshufd xmz, xmz, q3120
|
|
IF W, vpshufd xmw, xmw, q3120
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro conv16to32 0
|
|
op convert_U16_U32
|
|
LOAD_CONT tmp0q
|
|
IF X, vextracti128 xmx2, mx, 1
|
|
IF Y, vextracti128 xmy2, my, 1
|
|
IF Z, vextracti128 xmz2, mz, 1
|
|
IF W, vextracti128 xmw2, mw, 1
|
|
IF X, pmovzxwd mx, xmx
|
|
IF Y, pmovzxwd my, xmy
|
|
IF Z, pmovzxwd mz, xmz
|
|
IF W, pmovzxwd mw, xmw
|
|
IF X, pmovzxwd mx2, xmx2
|
|
IF Y, pmovzxwd my2, xmy2
|
|
IF Z, pmovzxwd mz2, xmz2
|
|
IF W, pmovzxwd mw2, xmw2
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro conv32to16 0
|
|
op convert_U32_U16
|
|
LOAD_CONT tmp0q
|
|
IF X, packusdw mx, mx2
|
|
IF Y, packusdw my, my2
|
|
IF Z, packusdw mz, mz2
|
|
IF W, packusdw mw, mw2
|
|
IF X, vpermq mx, mx, q3120
|
|
IF Y, vpermq my, my, q3120
|
|
IF Z, vpermq mz, mz, q3120
|
|
IF W, vpermq mw, mw, q3120
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Shifting
|
|
|
|
%macro lshift16 0
|
|
op lshift16
|
|
vmovq xm8, [implq + SwsOpImpl.priv]
|
|
LOAD_CONT tmp0q
|
|
IF X, psllw mx, xm8
|
|
IF Y, psllw my, xm8
|
|
IF Z, psllw mz, xm8
|
|
IF W, psllw mw, xm8
|
|
%if V2
|
|
IF X, psllw mx2, xm8
|
|
IF Y, psllw my2, xm8
|
|
IF Z, psllw mz2, xm8
|
|
IF W, psllw mw2, xm8
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
%macro rshift16 0
|
|
op rshift16
|
|
vmovq xm8, [implq + SwsOpImpl.priv]
|
|
LOAD_CONT tmp0q
|
|
IF X, psrlw mx, xm8
|
|
IF Y, psrlw my, xm8
|
|
IF Z, psrlw mz, xm8
|
|
IF W, psrlw mw, xm8
|
|
%if V2
|
|
IF X, psrlw mx2, xm8
|
|
IF Y, psrlw my2, xm8
|
|
IF Z, psrlw mz2, xm8
|
|
IF W, psrlw mw2, xm8
|
|
%endif
|
|
CONTINUE tmp0q
|
|
%endmacro
|
|
|
|
;---------------------------------------------------------
|
|
; Macro instantiations for kernel functions
|
|
|
|
%macro funcs_u8 0
|
|
read_planar 1
|
|
read_planar 2
|
|
read_planar 3
|
|
read_planar 4
|
|
write_planar 1
|
|
write_planar 2
|
|
write_planar 3
|
|
write_planar 4
|
|
|
|
rw_packed 8
|
|
read_nibbles
|
|
read_bits
|
|
write_bits
|
|
|
|
pack_generic 1, 2, 1
|
|
pack_generic 3, 3, 2
|
|
pack_generic 2, 3, 3
|
|
unpack8 1, 2, 1
|
|
unpack8 3, 3, 2
|
|
unpack8 2, 3, 3
|
|
|
|
clear_alpha 0, mx, mx2
|
|
clear_alpha 1, my, my2
|
|
clear_alpha 3, mw, mw2
|
|
clear_zero 0, mx, mx2
|
|
clear_zero 1, my, my2
|
|
clear_zero 3, mw, mw2
|
|
clear_funcs
|
|
swizzle_funcs
|
|
|
|
decl_common_patterns shuffle
|
|
%endmacro
|
|
|
|
%macro funcs_u16 0
|
|
rw_packed 16
|
|
pack_generic 4, 4, 4
|
|
pack_generic 5, 5, 5
|
|
pack_generic 5, 6, 5
|
|
unpack w, 16, 4, 4, 4
|
|
unpack w, 16, 5, 5, 5
|
|
unpack w, 16, 5, 6, 5
|
|
decl_common_patterns conv8to16 convert
|
|
decl_common_patterns conv8to16 expand
|
|
decl_common_patterns conv16to8
|
|
decl_common_patterns lshift16
|
|
decl_common_patterns rshift16
|
|
%endmacro
|
|
|
|
INIT_XMM sse4
|
|
decl_v2 0, funcs_u8
|
|
decl_v2 1, funcs_u8
|
|
|
|
INIT_YMM avx2
|
|
decl_v2 0, funcs_u8
|
|
decl_v2 1, funcs_u8
|
|
decl_v2 0, funcs_u16
|
|
decl_v2 1, funcs_u16
|
|
|
|
INIT_YMM avx2
|
|
decl_v2 1, rw_packed 32
|
|
decl_v2 1, pack_generic 10, 10, 10, 2
|
|
decl_v2 1, pack_generic 2, 10, 10, 10
|
|
decl_v2 1, unpack d, 32, 10, 10, 10, 2
|
|
decl_v2 1, unpack d, 32, 2, 10, 10, 10
|
|
decl_common_patterns conv8to32 convert
|
|
decl_common_patterns conv8to32 expand
|
|
decl_common_patterns conv32to8
|
|
decl_common_patterns conv16to32
|
|
decl_common_patterns conv32to16
|