virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-12-23 12:43:46 +02:00
Code Issues Releases Activity

aarch64: me_cmp: Don't do uaddlv once per iteration

Browse Source 
The max height is currently documented as 16; the max difference per
pixel is 255, and a .8h element can easily contain 16*255, thus keep
accumulating in two .8h vectors, and just do the final accumulationat the
end. This should work for heights up to 256.

This requires a minor register renumbering in ff_pix_abs16_xy2_neon.

Before:       Cortex A53    A72    A73   Graviton 3
pix_abs_0_0_neon:   97.7   47.0   37.5   22.7
pix_abs_0_1_neon:  154.0   59.0   52.0   25.0
pix_abs_0_3_neon:  179.7   96.7   87.5   41.2
After:
pix_abs_0_0_neon:   96.0   39.2   31.2   22.0
pix_abs_0_1_neon:  150.7   59.7   46.2   23.7
pix_abs_0_3_neon:  175.7   83.7   81.7   38.2

Signed-off-by: Martin Storsjö <martin@martin.st>
This commit is contained in:
Martin Storsjö 2022-07-13 00:48:48 +03:00
parent 68a03f6424
commit 4136405c86
1 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
libavcodec/aarch64/me_cmp_neon.S
View File

@ -27,15 +27,16 @@ function ff_pix_abs16_neon, export=1
// x3 ptrdiff_t stride // x3 ptrdiff_t stride
// w4 int h // w4 int h
cmp w4, #4 // if h < 4, jump to completion section cmp w4, #4 // if h < 4, jump to completion section
movi v18.4S, #0 // clear result accumulator movi v16.8h, #0 // clear result accumulator
movi v17.8h, #0 // clear result accumulator
b.lt 2f b.lt 2f
1: 1:
ld1 {v0.16b}, [x1], x3 // load pix1 ld1 {v0.16b}, [x1], x3 // load pix1
ld1 {v4.16b}, [x2], x3 // load pix2 ld1 {v4.16b}, [x2], x3 // load pix2
ld1 {v1.16b}, [x1], x3 // load pix1 ld1 {v1.16b}, [x1], x3 // load pix1
ld1 {v5.16b}, [x2], x3 // load pix2 ld1 {v5.16b}, [x2], x3 // load pix2
uabdl v16.8h, v0.8b, v4.8b // absolute difference accumulate uabal v16.8h, v0.8b, v4.8b // absolute difference accumulate
uabdl2 v17.8h, v0.16b, v4.16b uabal2 v17.8h, v0.16b, v4.16b
ld1 {v2.16b}, [x1], x3 // load pix1 ld1 {v2.16b}, [x1], x3 // load pix1
ld1 {v6.16b}, [x2], x3 // load pix2 ld1 {v6.16b}, [x2], x3 // load pix2
uabal v16.8h, v1.8b, v5.8b // absolute difference accumulate uabal v16.8h, v1.8b, v5.8b // absolute difference accumulate
@ -48,27 +49,26 @@ function ff_pix_abs16_neon, export=1
uabal v16.8h, v3.8b, v7.8b uabal v16.8h, v3.8b, v7.8b
uabal2 v17.8h, v3.16b, v7.16b uabal2 v17.8h, v3.16b, v7.16b
cmp w4, #4 // if h >= 4, loop cmp w4, #4 // if h >= 4, loop
add v16.8h, v16.8h, v17.8h
uaddlv s16, v16.8h // add up everything in v16 accumulator
add d18, d16, d18 // add to the end result register
b.ge 1b b.ge 1b
cbnz w4, 2f // if iterations remain, jump to completion section cbnz w4, 2f // if iterations remain, jump to completion section
fmov w0, s18 // copy result to general purpose register add v16.8h, v16.8h, v17.8h
uaddlv s16, v16.8h // add up everything in v16 accumulator
fmov w0, s16 // copy result to general purpose register
ret ret
2: 2:
ld1 {v0.16b}, [x1], x3 // load pix1 ld1 {v0.16b}, [x1], x3 // load pix1
ld1 {v4.16b}, [x2], x3 // load pix2 ld1 {v4.16b}, [x2], x3 // load pix2
uabdl v16.8h, v0.8b, v4.8b // absolute difference accumulate
uabal2 v16.8h, v0.16b, v4.16b
subs w4, w4, #1 // h -= 1 subs w4, w4, #1 // h -= 1
addv h16, v16.8h // add up v16 uabal v16.8h, v0.8b, v4.8b // absolute difference accumulate
add d18, d16, d18 // add to result uabal2 v17.8h, v0.16b, v4.16b
b.ne 2b b.ne 2b
fmov w0, s18 // copy result to general purpose register add v16.8h, v16.8h, v17.8h
uaddlv s16, v16.8h // add up everything in v16 accumulator
fmov w0, s16 // copy result to general purpose register
ret ret
endfunc endfunc
@ -80,7 +80,8 @@ function ff_pix_abs16_xy2_neon, export=1
// w4 int h // w4 int h
add x5, x2, x3 // use x5 to hold uint8_t *pix3 add x5, x2, x3 // use x5 to hold uint8_t *pix3
movi v0.2d, #0 // initialize the result register movi v21.8h, #0 // initialize the result register
movi v22.8h, #0 // initialize the result register
// Load initial pix2 values for either the unrolled version or completion version. // Load initial pix2 values for either the unrolled version or completion version.
ldur q4, [x2, #1] // load pix2+1 ldur q4, [x2, #1] // load pix2+1
@ -119,15 +120,15 @@ function ff_pix_abs16_xy2_neon, export=1
uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7 uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7
uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15 uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15
ldur q22, [x5, #1] // load pix3+1 ldur q7, [x5, #1] // load pix3+1
add v26.8h, v30.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above add v26.8h, v30.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above
add v27.8h, v31.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above add v27.8h, v31.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above
uabdl v24.8h, v1.8b, v23.8b // absolute difference 0..7, i=0 uabal v21.8h, v1.8b, v23.8b // absolute difference 0..7, i=0
uabdl2 v23.8h, v1.16b, v23.16b // absolute difference 8..15, i=0 uabal2 v22.8h, v1.16b, v23.16b // absolute difference 8..15, i=0
ld1 {v21.16b}, [x5], x3 // load pix3 ld1 {v6.16b}, [x5], x3 // load pix3
ld1 {v20.16b}, [x1], x3 // load pix1 ld1 {v20.16b}, [x1], x3 // load pix1
rshrn v26.8b, v26.8h, #2 // shift right 2 0..7 (rounding shift right) rshrn v26.8b, v26.8h, #2 // shift right 2 0..7 (rounding shift right)
@ -140,33 +141,33 @@ function ff_pix_abs16_xy2_neon, export=1
rshrn v28.8b, v28.8h, #2 // shift right 2 0..7 (rounding shift right) rshrn v28.8b, v28.8h, #2 // shift right 2 0..7 (rounding shift right)
rshrn2 v28.16b, v29.8h, #2 // shift right 2 8..15 rshrn2 v28.16b, v29.8h, #2 // shift right 2 8..15
uabal v24.8h, v16.8b, v26.8b // absolute difference 0..7, i=1 uabal v21.8h, v16.8b, v26.8b // absolute difference 0..7, i=1
uabal2 v23.8h, v16.16b, v26.16b // absolute difference 8..15, i=1 uabal2 v22.8h, v16.16b, v26.16b // absolute difference 8..15, i=1
uaddl v2.8h, v21.8b, v22.8b // pix3 + pix3+1 0..7 uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7
uaddl2 v3.8h, v21.16b, v22.16b // pix3 + pix3+1 8..15 uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15
add v30.8h, v4.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above add v30.8h, v4.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above
add v31.8h, v5.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above add v31.8h, v5.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above
rshrn v30.8b, v30.8h, #2 // shift right 2 0..7 (rounding shift right) rshrn v30.8b, v30.8h, #2 // shift right 2 0..7 (rounding shift right)
rshrn2 v30.16b, v31.8h, #2 // shift right 2 8..15 rshrn2 v30.16b, v31.8h, #2 // shift right 2 8..15
uabal v24.8h, v17.8b, v28.8b // absolute difference 0..7, i=2
uabal2 v23.8h, v17.16b, v28.16b // absolute difference 8..15, i=2
sub w4, w4, #4 // h -= 4 sub w4, w4, #4 // h -= 4
uabal v24.8h, v20.8b, v30.8b // absolute difference 0..7, i=3 uabal v21.8h, v17.8b, v28.8b // absolute difference 0..7, i=2
uabal2 v23.8h, v20.16b, v30.16b // absolute difference 8..15, i=3 uabal2 v22.8h, v17.16b, v28.16b // absolute difference 8..15, i=2
cmp w4, #4 // loop if h >= 4 cmp w4, #4 // loop if h >= 4
add v4.8h, v23.8h, v24.8h
uaddlv s4, v4.8h // finish adding up accumulated values uabal v21.8h, v20.8b, v30.8b // absolute difference 0..7, i=3
add d0, d0, d4 // add the value to the top level accumulator uabal2 v22.8h, v20.16b, v30.16b // absolute difference 8..15, i=3
b.ge 1b b.ge 1b
cbnz w4, 2f // if iterations remain jump to completion section cbnz w4, 2f // if iterations remain jump to completion section
add v4.8h, v21.8h, v22.8h
uaddlv s0, v4.8h // finish adding up accumulated values
fmov w0, s0 // copy result to general purpose register fmov w0, s0 // copy result to general purpose register
ret ret
2: 2:
@ -182,20 +183,18 @@ function ff_pix_abs16_xy2_neon, export=1
add v16.8h, v2.8h, v18.8h // add up 0..7, using pix2 + pix2+1 values from previous iteration add v16.8h, v2.8h, v18.8h // add up 0..7, using pix2 + pix2+1 values from previous iteration
add v17.8h, v3.8h, v19.8h // add up 8..15, using pix2 + pix2+1 values from previous iteration add v17.8h, v3.8h, v19.8h // add up 8..15, using pix2 + pix2+1 values from previous iteration
// divide by 4 to compute the average of values summed above // divide by 4 to compute the average of values summed above
urshr v16.8h, v16.8h, #2 // shift right by 2 0..7 (rounding shift right) rshrn v16.8b, v16.8h, #2 // shift right by 2 0..7 (rounding shift right)
urshr v17.8h, v17.8h, #2 // shift right by 2 8..15 rshrn2 v16.16b, v17.8h, #2 // shift right by 2 8..15
uxtl2 v7.8h, v1.16b // 8->16 bits pix1 8..15 uabal v21.8h, v1.8b, v16.8b // absolute difference 0..7
uxtl v1.8h, v1.8b // 8->16 bits pix1 0..7 uabal2 v22.8h, v1.16b, v16.16b // absolute difference accumulate 8..15
uabd v6.8h, v1.8h, v16.8h // absolute difference 0..7
uaba v6.8h, v7.8h, v17.8h // absolute difference accumulate 8..15
mov v2.16b, v18.16b // pix3 -> pix2 mov v2.16b, v18.16b // pix3 -> pix2
mov v3.16b, v19.16b // pix3+1 -> pix2+1 mov v3.16b, v19.16b // pix3+1 -> pix2+1
uaddlv s6, v6.8h // add up accumulator in v6
add d0, d0, d6 // add to the final result
b.ne 2b // loop if h > 0 b.ne 2b // loop if h > 0
add v4.8h, v21.8h, v22.8h
uaddlv s0, v4.8h // finish adding up accumulated values
fmov w0, s0 // copy result to general purpose register fmov w0, s0 // copy result to general purpose register
ret ret
endfunc endfunc
@ -209,7 +208,8 @@ function ff_pix_abs16_x2_neon, export=1
cmp w4, #4 cmp w4, #4
// initialize buffers // initialize buffers
movi d20, #0 movi v16.8h, #0
movi v17.8h, #0
add x5, x2, #1 // pix2 + 1 add x5, x2, #1 // pix2 + 1
b.lt 2f b.lt 2f
@ -224,9 +224,9 @@ function ff_pix_abs16_x2_neon, export=1
ld1 {v2.16b}, [x5], x3 ld1 {v2.16b}, [x5], x3
urhadd v30.16b, v1.16b, v2.16b urhadd v30.16b, v1.16b, v2.16b
ld1 {v0.16b}, [x1], x3 ld1 {v0.16b}, [x1], x3
uabdl v16.8h, v0.8b, v30.8b uabal v16.8h, v0.8b, v30.8b
ld1 {v4.16b}, [x2], x3 ld1 {v4.16b}, [x2], x3
uabdl2 v17.8h, v0.16b, v30.16b uabal2 v17.8h, v0.16b, v30.16b
ld1 {v5.16b}, [x5], x3 ld1 {v5.16b}, [x5], x3
urhadd v29.16b, v4.16b, v5.16b urhadd v29.16b, v4.16b, v5.16b
ld1 {v3.16b}, [x1], x3 ld1 {v3.16b}, [x1], x3
@ -238,20 +238,15 @@ function ff_pix_abs16_x2_neon, export=1
ld1 {v6.16b}, [x1], x3 ld1 {v6.16b}, [x1], x3
uabal v16.8h, v6.8b, v28.8b uabal v16.8h, v6.8b, v28.8b
ld1 {v24.16b}, [x2], x3 ld1 {v24.16b}, [x2], x3
sub w4, w4, #4
uabal2 v17.8h, v6.16b, v28.16b uabal2 v17.8h, v6.16b, v28.16b
ld1 {v25.16b}, [x5], x3 ld1 {v25.16b}, [x5], x3
urhadd v27.16b, v24.16b, v25.16b urhadd v27.16b, v24.16b, v25.16b
ld1 {v23.16b}, [x1], x3 ld1 {v23.16b}, [x1], x3
cmp w4, #4
uabal v16.8h, v23.8b, v27.8b uabal v16.8h, v23.8b, v27.8b
uabal2 v17.8h, v23.16b, v27.16b uabal2 v17.8h, v23.16b, v27.16b
sub w4, w4, #4
add v16.8h, v16.8h, v17.8h
uaddlv s16, v16.8h
cmp w4, #4
add d20, d20, d16
b.ge 1b b.ge 1b
cbz w4, 3f cbz w4, 3f
@ -259,18 +254,19 @@ function ff_pix_abs16_x2_neon, export=1
2: 2:
ld1 {v1.16b}, [x2], x3 ld1 {v1.16b}, [x2], x3
ld1 {v2.16b}, [x5], x3 ld1 {v2.16b}, [x5], x3
subs w4, w4, #1
urhadd v29.16b, v1.16b, v2.16b urhadd v29.16b, v1.16b, v2.16b
ld1 {v0.16b}, [x1], x3 ld1 {v0.16b}, [x1], x3
uabd v28.16b, v0.16b, v29.16b uabal v16.8h, v0.8b, v29.8b
uabal2 v17.8h, v0.16b, v29.16b
uaddlv h28, v28.16b
subs w4, w4, #1
add d20, d20, d28
b.ne 2b b.ne 2b
3: 3:
fmov w0, s20 add v16.8h, v16.8h, v17.8h
uaddlv s16, v16.8h
fmov w0, s16
ret ret
endfunc endfunc