x/ffmpeg
1
0
Fork 0
mirror of https://git.ffmpeg.org/ffmpeg.git synced 2026-08-12 13:23:48 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
ef82d0789afe9783cf2aad186addbe81fa30d00a
ffmpeg/libswscale/x86/ops_int.asm
T
Niklas Haas 44e1b60c6a swscale/uops: consolidate SWS_UOP_{PERMUTE,COPY,MOVE} 
Rather than having two uops for the C reference code and a separate
uop for the x86 backend, we can just implement the C reference
using the same register-register assignment logic.

This results in only slightly worse codegen, since the compiler is
smart enough to eliminate the unnecessary copies.

The main downside is the fact that we need to allocate extra stack space
to back-fill the remaining (unused) pointers. In theory, we could do this
more efficiently by re-using unused *input* registers directly, but that
would require a lot of reliance on compile-time constant folding teasing
apart the entire register index tracking loop and duplicate check, so this
is a useful middle ground.

On platforms like x86, we still don't care about the PERMUTE vs
COPY distinction, but it's also harmless (2-3 extra lines to
instantiate the underlying MOVE macro on both uop variants).

Finally, we need to add a mask to the move uops to keep track of which
outputs are actually needed (especially for checkasm). Though it does result
in the rather unfortunate downside that it bloats the names of the uop
names.

The op lists change to reflect the new representation, e.g.:

 rgb24 16x16 -> bgr24 16x16:
   [ u8 +++X] SWS_OP_READ         : 3 elem(s) packed >> 0
     min: {0 0 0 _}, max: {255 255 255 _}
   [ u8 +++X] SWS_OP_SWIZZLE      : 2103
     min: {0 0 0 _}, max: {255 255 255 _}
   [ u8 XXXX] SWS_OP_WRITE        : 3 elem(s) packed >> 0
     ('X' unused, 'z' byteswapped, '=' copied, '$' const, '+' integer, '0' zero)
  translated micro-ops:
     u8_read_packed_xyz
-    u8_permute_zyxw
+    u8_permute_xyzw_txz_xzt
     u8_write_packed_xyz

Signed-off-by: Niklas Haas <git@haasn.dev>
2026-07-14 12:24:01 +00:00

848 lines
23 KiB
NASM
Raw Blame History

;******************************************************************************
;* Copyright (c) 2025-2026 Niklas Haas
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "ops_include.asm"
SECTION_RODATA
align 16
read8_unpack2: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15
read8_unpack3: db 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, -1, -1, -1, -1
read8_unpack4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15
read16_unpack2: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
read16_unpack3: db 0, 1, 6, 7, 2, 3, 8, 9, 4, 5, 10, 11, -1, -1, -1, -1
read16_unpack4: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
write8_pack2: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
write8_pack3: db 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, -1, -1, -1, -1
write16_pack3: db 0, 1, 4, 5, 8, 9, 2, 3, 6, 7, 10, 11, -1, -1, -1, -1
%define write8_pack4 read8_unpack4
%define write16_pack4 read16_unpack2
%define write16_pack2 read16_unpack4
bits_mask: db 128, 64, 32, 16, 8, 4, 2, 1,128, 64, 32, 16, 8, 4, 2, 1
bits_reverse: db 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
nibbles_pack: times 8 dw 0x0110
swap16: db 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14
swap32: db 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
expand16: db 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14
expand32: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12
align 32
bits_shuf: times 8 db 0
times 8 db 1
times 8 db 2
times 8 db 3
mask1: times 32 db 0x01
mask2: times 32 db 0x03
mask3: times 32 db 0x07
mask4: times 32 db 0x0F
const1b equ mask1
const1w: times 16 dw 0x1
SECTION .text
;---------------------------------------------------------
; Helper macros for BITS-dependent sized instructions
%macro bitfn 2+
%if BITS == 8
%1%+b %2
%elif BITS == 16
%1%+w %2
%elif BITS == 32
%1%+d %2
%endif
%endmacro
%define psllx bitfn psll,
%define psrlx bitfn psrl,
%define pcmpeqx bitfn pcmpeq,
%define pmullx bitfn pmull,
%define vpbroadcastx bitfn vpbroadcast,
;---------------------------------------------------------
; Planar reads / writes
%macro READ_PLANAR 0
IF X, movu mx, [in0q]
IF Y, movu my, [in1q]
IF Z, movu mz, [in2q]
IF W, movu mw, [in3q]
%if V2
IF X, movu mx2, [in0q + mmsize]
IF Y, movu my2, [in1q + mmsize]
IF Z, movu mz2, [in2q + mmsize]
IF W, movu mw2, [in3q + mmsize]
%endif
LOAD_CONT tmp0q
IF X, add in0q, BLOCK_SIZE
IF Y, add in1q, BLOCK_SIZE
IF Z, add in2q, BLOCK_SIZE
IF W, add in3q, BLOCK_SIZE
CONTINUE tmp0q
%endmacro
%macro READ_NIBBLE 0
assert COMPS == 1
VBROADCASTI128 m12, [mask4]
LOAD_CONT tmp0q
pmovzxbw mx, [in0q]
IF V2, pmovzxbw mx2, [in0q + (mmsize >> 1)]
add in0q, BLOCK_WIDTH >> 1
psllw m8, mx, 8 ; { 00 AB 00 CD ... }
psrlw mx, 4 ; { 0A 00 0C 00 ... }
pand m8, m12 ; { 00 0B 00 0D ... }
por mx, m8 ; { 0A 0B 0C 0D ... }
%if V2
psllw m9, mx2, 8
psrlw mx2, 4
pand m9, m12
por mx2, m9
%endif
CONTINUE tmp0q
%endmacro
%macro WRITE_NIBBLE 0
assert COMPS == 1
VBROADCASTI128 m12, [nibbles_pack]
pmaddubsw mx, m12
packuswb mx, mx
%if V2
pmaddubsw mx2, m12
packuswb mx2, mx2
%endif
%if cpuflag(avx2)
vpermq mx, mx, q3120
IF V2, vpermq mx2, mx2, q3120
%endif
movu [out0q], xmx
IF V2, movu [out0q + (mmsize >> 1)], xmx2
add out0q, BLOCK_WIDTH >> 1
RET
%endmacro
%macro READ_BIT 0
assert COMPS == 1
%if cpuflag(avx2)
vpbroadcastd mx, [in0q]
IF V2, vpbroadcastd mx2, [in0q + 4]
%else
movd mx, [in0q]
IF V2, movd mx2, [in0q + 2]
%endif
mova m8, [bits_shuf]
VBROADCASTI128 m9, [bits_mask]
VBROADCASTI128 m10, [const1b]
LOAD_CONT tmp0q
add in0q, BLOCK_WIDTH >> 3
pshufb mx, m8
IF V2, pshufb mx2, m8
pand mx, m9
IF V2, pand mx2, m9
pcmpeqb mx, m9
IF V2, pcmpeqb mx2, m9
pand mx, m10
IF V2, pand mx2, m10
CONTINUE tmp0q
%endmacro
%macro WRITE_BIT 0
assert COMPS == 1
VBROADCASTI128 m8, [bits_reverse]
psllw mx, 7
IF V2, psllw mx2, 7
pshufb mx, m8
IF V2, pshufb mx2, m8
pmovmskb tmp0d, mx
IF V2, pmovmskb tmp1d, mx2
%if mmsize >= 32
mov [out0q], tmp0d
IF V2, mov [out0q + (mmsize >> 3)], tmp1d
%else
mov [out0q], tmp0w
IF V2, mov [out0q + (mmsize >> 3)], tmp1w
%endif
add out0q, BLOCK_WIDTH >> 3
RET
%endmacro
%macro WRITE_PLANAR 0
IF X, movu [out0q], mx
IF Y, movu [out1q], my
IF Z, movu [out2q], mz
IF W, movu [out3q], mw
%if V2
IF X, movu [out0q + mmsize], mx2
IF Y, movu [out1q + mmsize], my2
IF Z, movu [out2q + mmsize], mz2
IF W, movu [out3q + mmsize], mw2
%endif
IF X, add out0q, BLOCK_WIDTH
IF Y, add out1q, BLOCK_WIDTH
IF Z, add out2q, BLOCK_WIDTH
IF W, add out3q, BLOCK_WIDTH
RET
%endmacro
;---------------------------------------------------------
; Packed reads, packing and unpacking
%macro read_packed2 0
movu m8, [in0q + 0 * mmsize]
movu m9, [in0q + 1 * mmsize]
IF V2, movu m10, [in0q + 2 * mmsize]
IF V2, movu m11, [in0q + 3 * mmsize]
%if BITS == 32
shufps m8, m8, q3120
shufps m9, m9, q3120
IF V2, shufps m10, m10, q3120
IF V2, shufps m11, m11, q3120
%else
pshufb m8, m12 ; { X0 Y0 | X1 Y1 }
pshufb m9, m12 ; { X2 Y2 | X3 Y3 }
IF V2, pshufb m10, m12
IF V2, pshufb m11, m12
%endif
unpcklpd mx, m8, m9 ; { X0 X2 | X1 X3 }
unpckhpd my, m8, m9 ; { Y0 Y2 | Y1 Y3 }
IF V2, unpcklpd mx2, m10, m11
IF V2, unpckhpd my2, m10, m11
%if cpuflag(avx2)
vpermq mx, mx, q3120 ; { X0 X1 | X2 X3 }
vpermq my, my, q3120 ; { Y0 Y1 | Y2 Y3 }
IF V2, vpermq mx2, mx2, q3120
IF V2, vpermq my2, my2, q3120
%endif
%endmacro
%macro read_packed34 5 ; x, y, z, w, addr
movu xm8, [%5 + 0 * COMPS]
movu xm9, [%5 + 4 * COMPS]
movu xm10, [%5 + 8 * COMPS]
movu xm11, [%5 + 12 * COMPS]
%if cpuflag(avx2)
vinserti128 m8, m8, [%5 + 16 * COMPS], 1
vinserti128 m9, m9, [%5 + 20 * COMPS], 1
vinserti128 m10, m10, [%5 + 24 * COMPS], 1
vinserti128 m11, m11, [%5 + 28 * COMPS], 1
%endif
%if BITS == 32
mova %1, m8
mova %2, m9
mova %3, m10
mova %4, m11
%else
pshufb %1, m8, m12 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
pshufb %2, m9, m12 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
pshufb %3, m10, m12 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
pshufb %4, m11, m12 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
%endif
punpckldq m8, %1, %2 ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 }
punpckldq m9, %3, %4 ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 }
punpckhdq m10, %1, %2 ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 }
punpckhdq m11, %3, %4 ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 }
punpcklqdq %1, m8, m9 ; { X0 X1 X2 X3 | X4 X5 X6 X7 }
punpckhqdq %2, m8, m9 ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 }
punpcklqdq %3, m10, m11 ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 }
IF W, punpckhqdq %4, m10, m11 ; { W0 W1 W2 W3 | W4 W5 W6 W7 }
%endmacro
%macro READ_PACKED 0
%if BITS < 32
VBROADCASTI128 m12, [ read %+ BITS %+ _unpack %+ COMPS ]
%endif
LOAD_CONT tmp0q
%if COMPS == 2
read_packed2
%else
read_packed34 mx, my, mz, mw, in0q
IF1 V2, read_packed34 mx2, my2, mz2, mw2, in0q + mmsize * COMPS
%endif
add in0q, BLOCK_SIZE * COMPS
CONTINUE tmp0q
%endmacro
%macro read_pal8 6 ; x, y, z, w, palette, index
%if cpuflag(avx2)
pmovzxbd %1, [%6 + 0]
pmovzxbd %2, [%6 + 8]
pmovzxbd %3, [%6 + 16]
pmovzxbd %4, [%6 + 24]
vperm2i128 m8, %1, %3, q0200
vperm2i128 m9, %1, %3, q0301
vperm2i128 m10, %2, %4, q0200
vperm2i128 m11, %2, %4, q0301
pcmpeqb m14, m14
pcmpeqb m15, m15
vpgatherdd %1, [%5 + 4 * m8], m14
vpgatherdd %2, [%5 + 4 * m9], m15
pcmpeqb m14, m14
pcmpeqb m15, m15
vpgatherdd %3, [%5 + 4 * m10], m14
vpgatherdd %4, [%5 + 4 * m11], m15
pshufb %1, m12
pshufb %2, m12
pshufb %3, m12
pshufb %4, m12
punpckldq m8, %1, %2
punpckldq m9, %3, %4
punpckhdq m10, %1, %2
punpckhdq m11, %3, %4
punpcklqdq %1, m8, m9
punpckhqdq %2, m8, m9
punpcklqdq %3, m10, m11
punpckhqdq %4, m10, m11
%else ; !cpuflag(avx2)
%assign i 0
%rep 16
movzx tmp1d, byte [%6 + i]
pinsrb %1, [%5 + 4 * tmp1q + 0], i
pinsrb %2, [%5 + 4 * tmp1q + 1], i
pinsrb %3, [%5 + 4 * tmp1q + 2], i
pinsrb %4, [%5 + 4 * tmp1q + 3], i
%assign i i+1
%endrep
%endif
%endmacro
%macro READ_PALETTE 0
assert COMPS == 4
assert BITS == 8
%if cpuflag(avx2)
VBROADCASTI128 m12, [read8_unpack4]
%endif
LOAD_CONT tmp0q
read_pal8 mx, my, mz, mw, in1q, in0q
IF1 V2, read_pal8 mx2, my2, mz2, mw2, in1q, in0q + mmsize
add in0q, BLOCK_SIZE
CONTINUE tmp0q
%endmacro
%macro write_packed2 0
%if cpuflag(avx2)
vpermq mx, mx, q3120 ; { X0 X2 | X1 X3 }
vpermq my, my, q3120 ; { Y0 Y2 | Y1 Y3 }
IF V2, vpermq mx2, mx2, q3120
IF V2, vpermq my2, my2, q3120
%endif
unpcklpd m8, mx, my ; { X0 Y0 | X1 Y1 }
unpckhpd m9, mx, my ; { X2 Y2 | X3 Y3 }
IF V2, unpcklpd m10, mx2, my2
IF V2, unpckhpd m11, mx2, my2
%if BITS == 32
shufps m8, m8, q3120
shufps m9, m9, q3120
IF V2, shufps m10, m10, q3120
IF V2, shufps m11, m11, q3120
%else
pshufb m8, m12
pshufb m9, m12
IF V2, pshufb m10, m12
IF V2, pshufb m11, m12
%endif
movu [out0q + 0 * mmsize], m8
movu [out0q + 1 * mmsize], m9
IF V2, movu [out0q + 2 * mmsize], m10
IF V2, movu [out0q + 3 * mmsize], m11
%endmacro
%macro write_packed34 5 ; x, y, z, w, addr
punpckldq m8, %1, %2 ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 }
punpckldq m9, %3, %4 ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 }
punpckhdq m10, %1, %2 ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 }
punpckhdq m11, %3, %4 ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 }
punpcklqdq %1, m8, m9 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
punpckhqdq %2, m8, m9 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
punpcklqdq %3, m10, m11 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
punpckhqdq %4, m10, m11 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
%if BITS == 32
mova m8, %1
mova m9, %2
mova m10, %3
mova m11, %4
%else
pshufb m8, %1, m12
pshufb m9, %2, m12
pshufb m10, %3, m12
pshufb m11, %4, m12
%endif
movu [%5 + 0 * COMPS], xm8
movu [%5 + 4 * COMPS], xm9
movu [%5 + 8 * COMPS], xm10
movu [%5 + 12 * COMPS], xm11
%if cpuflag(avx2)
vextracti128 [%5 + 16 * COMPS], m8, 1
vextracti128 [%5 + 20 * COMPS], m9, 1
vextracti128 [%5 + 24 * COMPS], m10, 1
vextracti128 [%5 + 28 * COMPS], m11, 1
%endif
%endmacro
%macro WRITE_PACKED 0
%if BITS < 32
VBROADCASTI128 m12, [ write %+ BITS %+ _pack %+ COMPS ]
%endif
LOAD_CONT tmp0q
%if COMPS == 2
write_packed2
%else
write_packed34 mx, my, mz, mw, out0q
IF1 V2, write_packed34 mx2, my2, mz2, mw2, out0q + mmsize * COMPS
%endif
add out0q, BLOCK_SIZE * COMPS
RET
%endmacro
%macro PACK 4 ; x, y, z, w
; pslld works for all sizes because the input should not overflow
IF %2, pslld mx, %4+%3+%2
IF %3, pslld my, %4+%3
IF %4, pslld mz, %4
IF Y, por mx, my
IF Z, por mx, mz
IF W, por mx, mw
%if V2
IF %2, pslld mx2, %4+%3+%2
IF %3, pslld my2, %4+%3
IF %4, pslld mz2, %4
IF Y, por mx2, my2
IF Z, por mx2, mz2
IF W, por mx2, mw2
%endif
CONTINUE
%endmacro
%macro UNPACK 4 ; x, y, z, w
LOAD_CONT tmp0q
%if BITS == 8
assert MASK == SWS_COMP_ELEMS(3)
pand mz, mx, [mask%3]
psrld my, mx, %3
psrld mx, %3+%2
pand my, [mask%2]
pand mx, [mask%1]
%if V2
pand mz2, mx2, [mask%3]
psrld my2, mx2, %3
psrld mx2, %3+%2
pand my2, [mask%2]
pand mx2, [mask%1]
%endif
%else
; clear high bits by shifting left
IF W, psllx mw, mx, BITS - (%4)
IF Z, psllx mz, mx, BITS - (%4+%3)
IF Y, psllx my, mx, BITS - (%4+%3+%2)
psrlx mx, %2+%3+%4
IF Y, psrlx my, BITS - %2
IF Z, psrlx mz, BITS - %3
IF W, psrlx mw, BITS - %4
%if V2
IF W, psllx mw2, mx2, BITS - (%4)
IF Z, psllx mz2, mx2, BITS - (%4+%3)
IF Y, psllx my2, mx2, BITS - (%4+%3+%2)
psrlx mx2, %2+%3+%4
IF Y, psrlx my2, BITS - %2
IF Z, psrlx mz2, BITS - %3
IF W, psrlx mw2, BITS - %4
%endif
%endif
CONTINUE tmp0q
%endmacro
;---------------------------------------------------------
; Pixel type conversions
%macro cast8to16 4 ; reg, reg2, xreg, xreg2
IF V2, vextracti128 %4, %1, 1
pmovzxbw %1, %3
IF V2, pmovzxbw %2, %4
%endmacro
%macro cast16to8 4 ; reg, reg2, xreg, xreg2
%if V2
packuswb %1, %2
vpermq %1, %1, q3120
%else
vextracti128 %4, %1, 1
packuswb %3, %4
%endif
%endmacro
%macro cast8to32 4 ; reg, reg2, xreg, xreg2
psrldq %4, %3, 8
pmovzxbd %1, %3
pmovzxbd %2, %4
%endmacro
%macro cast32to8 4 ; reg, reg2, xreg, xreg2
packusdw %1, %2
vextracti128 %4, %1, 1
packuswb %3, %4
vpshufd %3, %3, q3120
%endmacro
%macro cast16to32 4 ; reg, reg2, xreg, xreg2
vextracti128 %4, %1, 1
pmovzxwd %1, %3
pmovzxwd %2, %4
%endmacro
%macro cast32to16 4 ; reg, reg2, xreg, xreg2
packusdw %1, %2
vpermq %1, %1, q3120
%endmacro
%macro CAST_TO 0
%ifidn UOP, SWS_UOP_TO_U8
%assign BITS_TO 8
%elifidn UOP, SWS_UOP_TO_U16
%assign BITS_TO 16
%else
%assign BITS_TO 32
%endif
LOAD_CONT tmp0q
%ifidn TYPE, SWS_PIXEL_F32
IF X, cvttps2dq mx, mx
IF Y, cvttps2dq my, my
IF Z, cvttps2dq mz, mz
IF W, cvttps2dq mw, mw
IF X, cvttps2dq mx2, mx2
IF Y, cvttps2dq my2, my2
IF Z, cvttps2dq mz2, mz2
IF W, cvttps2dq mw2, mw2
%endif
%if BITS != BITS_TO
; integer size conversion
IF1 X, cast %+ BITS %+ to %+ BITS_TO mx, mx2, xmx, xmx2
IF1 Y, cast %+ BITS %+ to %+ BITS_TO my, my2, xmy, xmy2
IF1 Z, cast %+ BITS %+ to %+ BITS_TO mz, mz2, xmz, xmz2
IF1 W, cast %+ BITS %+ to %+ BITS_TO mw, mw2, xmw, xmw2
%if cpuflag(avx2) && (BITS > BITS_TO && !V2 || BITS >= BITS_TO * 4)
; clear upper bits after reducing the register size
vzeroupper ; TMP
%endif
%endif
%ifidn UOP, SWS_UOP_TO_F32
IF X, vcvtdq2ps mx, mx
IF Y, vcvtdq2ps my, my
IF Z, vcvtdq2ps mz, mz
IF W, vcvtdq2ps mw, mw
IF X, vcvtdq2ps mx2, mx2
IF Y, vcvtdq2ps my2, my2
IF Z, vcvtdq2ps mz2, mz2
IF W, vcvtdq2ps mw2, mw2
%endif
CONTINUE tmp0q
%endmacro
;---------------------------------------------------------
; Moving, copying and clearing
%macro MOVE 13 ; num, dst0..dst5, src0..src5
%assign NUM_MOVES %1
%define DST %2
%define SRC %8
LOAD_CONT tmp0q
%rep NUM_MOVES
%assign dstidx %2 < 0 ? 8 : %2
%assign srcidx %8 < 0 ? 8 : %8
mova m %+ dstidx, m %+ srcidx
%if V2
%assign dstidx dstidx + 4
%assign srcidx srcidx + 4
mova m %+ dstidx, m %+ srcidx
%endif
%rotate 1
%endrep
CONTINUE tmp0q
%endmacro
%macro clear 3 ; idx, reg, reg2
%if SWS_COMP_TEST(ZERO_MASK, %1)
pxor %2, %2
%elif SWS_COMP_TEST(ONE_MASK, %1)
pcmpeqb %2, %2
%elif cpuflag(avx)
vpbroadcastd %2, [implq + SwsOpImpl.priv + 4 * %1]
%else
movd %2, [implq + SwsOpImpl.priv + 4 * %1]
pshufd %2, %2, 0
%endif
IF V2, mova %3, %2
%endmacro
%macro CLEAR 2
%assign ONE_MASK %1
%assign ZERO_MASK %2
IF1 X, clear 0, mx, mx2
IF1 Y, clear 1, my, my2
IF1 Z, clear 2, mz, mz2
IF1 W, clear 3, mw, mw2
CONTINUE
%endmacro
;---------------------------------------------------------
; Bit manipulation
%macro SWAP_BYTES 0
VBROADCASTI128 m8, [swap %+ BITS]
IF X, pshufb mx, m8
IF Y, pshufb my, m8
IF Z, pshufb mz, m8
IF W, pshufb mw, m8
%if V2
IF X, pshufb mx2, m8
IF Y, pshufb my2, m8
IF Z, pshufb mz2, m8
IF W, pshufb mw2, m8
%endif
CONTINUE
%endmacro
%macro EXPAND_BIT 0
%if BITS == 8
VBROADCASTI128 m8, [const1b]
%elif BITS == 16
VBROADCASTI128 m8, [const1w]
%else
assert 0, EXPAND_BIT is not implemented for 32-bit types
%endif
IF X, pcmpeqx mx, m8
IF Y, pcmpeqx my, m8
IF Z, pcmpeqx mz, m8
IF W, pcmpeqx mw, m8
%if V2
IF X, pcmpeqx mx2, m8
IF Y, pcmpeqx my2, m8
IF Z, pcmpeqx mz2, m8
IF W, pcmpeqx mw2, m8
%endif
CONTINUE
%endmacro
%macro EXPAND_BYTE 0
assert BITS == 8
%ifidn UOP, SWS_UOP_EXPAND_PAIR
%assign BITS_TO BITS * 2
%elifidn UOP, SWS_UOP_EXPAND_QUAD
%assign BITS_TO BITS * 4
%endif
VBROADCASTI128 m8, [expand %+ BITS_TO]
LOAD_CONT tmp0q
IF1 X, cast8to %+ BITS_TO mx, mx2, xmx, xmx2
IF1 Y, cast8to %+ BITS_TO my, my2, xmy, xmy2
IF1 Z, cast8to %+ BITS_TO mz, mz2, xmz, xmz2
IF1 W, cast8to %+ BITS_TO mw, mw2, xmw, xmw2
IF X, pshufb mx, m8
IF Y, pshufb my, m8
IF Z, pshufb mz, m8
IF W, pshufb mw, m8
%if V2
IF X, pshufb mx2, m8
IF Y, pshufb my2, m8
IF Z, pshufb mz2, m8
IF W, pshufb mw2, m8
%endif
CONTINUE tmp0q
%endmacro
%macro LSHIFT 1 ; shift
LOAD_CONT tmp0q
IF X, psllx mx, %1
IF Y, psllx my, %1
IF Z, psllx mz, %1
IF W, psllx mw, %1
%if V2
IF X, psllx mx2, %1
IF Y, psllx my2, %1
IF Z, psllx mz2, %1
IF W, psllx mw2, %1
%endif
CONTINUE tmp0q
%endmacro
%macro RSHIFT 1 ; shift
LOAD_CONT tmp0q
IF X, psrlx mx, %1
IF Y, psrlx my, %1
IF Z, psrlx mz, %1
IF W, psrlx mw, %1
%if V2
IF X, psrlx mx2, %1
IF Y, psrlx my2, %1
IF Z, psrlx mz2, %1
IF W, psrlx mw2, %1
%endif
CONTINUE tmp0q
%endmacro
;---------------------------------------------------------
; Arithmetic operations
%macro pmull4 6 ; suffix, x, y, z, w, k
IF X, pmull%1 %2, %6
IF Y, pmull%1 %3, %6
IF Z, pmull%1 %4, %6
IF W, pmull%1 %5, %6
%endmacro
%macro scale8 5 ; mx, my, mz, mw, k
IF X, mova m8, %1
IF Y, mova m9, %2
IF Z, mova m10, %3
IF W, mova m11, %4
IF X, punpcklbw %1, m15
IF Y, punpcklbw %2, m15
IF Z, punpcklbw %3, m15
IF W, punpcklbw %4, m15
IF X, punpckhbw m8, m15
IF Y, punpckhbw m9, m15
IF Z, punpckhbw m10, m15
IF W, punpckhbw m11, m15
pmull4 w, %1, %2, %3, %4, m12
pmull4 w, m8, m9, m10, m11, m12
IF X, packuswb %1, m8
IF Y, packuswb %2, m9
IF Z, packuswb %3, m10
IF W, packuswb %4, m11
%endmacro
%macro SCALE 0
LOAD_CONT tmp0q
%if BITS == 8
%if cpuflag(avx2)
vpbroadcastw m12, [implq + SwsOpImpl.priv]
%else
movd xm12, [implq + SwsOpImpl.priv]
SPLATW m12, xm12, 0
%endif
pxor m15, m15
scale8 mx, my, mz, mw, m12
IF1 V2, scale8 mx2, my2, mz2, mw2, m12
%else
vpbroadcastx m12, [implq + SwsOpImpl.priv]
pmull4 x, mx, my, mz, mw, m12
IF1 V2, pmull4 x, mx2, my2, mz2, mw2, m12
%endif
CONTINUE tmp0q
%endmacro
%macro ADD 0
assert 0, SWS_UOP_ADD is not implemented for integer types
%endmacro
%macro MIN 0
assert 0, SWS_UOP_MIN is not implemented for integer types
%endmacro
%macro MAX 0
assert 0, SWS_UOP_MAX is not implemented for integer types
%endmacro
%macro LINEAR_FMA 0-*
assert 0, SWS_UOP_LINEAR_FMA is not implemented for integer types
%endmacro
%macro DITHER 0-*
assert 0, SWS_UOP_DITHER is not implemented for integer types
%endmacro
;---------------------------------------------------------
; Instantiate above macros to generate all uop kernels
%macro decl_ops 1 ; type
DECL_%1_READ_PACKED (READ_PACKED)
DECL_%1_READ_NIBBLE (READ_NIBBLE)
DECL_%1_READ_BIT (READ_BIT)
DECL_%1_READ_PALETTE (READ_PALETTE)
DECL_%1_WRITE_PACKED (WRITE_PACKED)
DECL_%1_WRITE_NIBBLE (WRITE_NIBBLE)
DECL_%1_WRITE_BIT (WRITE_BIT)
DECL_%1_PERMUTE (MOVE)
DECL_%1_COPY (MOVE)
DECL_%1_SWAP_BYTES (SWAP_BYTES)
DECL_%1_EXPAND_BIT (EXPAND_BIT)
DECL_%1_SCALE (SCALE)
DECL_%1_ADD (ADD)
DECL_%1_MIN (MIN)
DECL_%1_MAX (MAX)
DECL_%1_UNPACK (UNPACK)
DECL_%1_PACK (PACK)
DECL_%1_LSHIFT (LSHIFT)
DECL_%1_RSHIFT (RSHIFT)
DECL_%1_LINEAR_FMA (LINEAR_FMA)
DECL_%1_DITHER (DITHER)
%endmacro
%macro decl_type_invariant 0
DECL_U8_READ_PLANAR (READ_PLANAR)
DECL_U8_WRITE_PLANAR (WRITE_PLANAR)
DECL_U8_CLEAR (CLEAR)
%endmacro
%macro decl_cast_u16 0
DECL_U8_TO_U16 (CAST_TO)
DECL_U16_TO_U8 (CAST_TO)
DECL_U8_EXPAND_PAIR (EXPAND_BYTE)
%endmacro
%macro decl_cast_u32 0
DECL_U8_TO_U32 (CAST_TO)
DECL_U32_TO_U8 (CAST_TO)
DECL_U16_TO_U32 (CAST_TO)
DECL_U32_TO_U16 (CAST_TO)
DECL_U8_EXPAND_QUAD (EXPAND_BYTE)
%endmacro
%macro decl_cast_f32 0
DECL_U8_TO_F32 (CAST_TO)
DECL_F32_TO_U8 (CAST_TO)
DECL_U16_TO_F32 (CAST_TO)
DECL_F32_TO_U16 (CAST_TO)
DECL_U32_TO_F32 (CAST_TO)
DECL_F32_TO_U32 (CAST_TO)
%endmacro
INIT_XMM sse4
decl_v2 0, decl_ops U8
decl_v2 1, decl_ops U8
decl_v2 0, decl_type_invariant
decl_v2 1, decl_type_invariant
INIT_YMM avx2
decl_v2 0, decl_ops U8
decl_v2 1, decl_ops U8
decl_v2 0, decl_ops U16
decl_v2 1, decl_ops U16
decl_v2 0, decl_type_invariant
decl_v2 1, decl_type_invariant
decl_v2 0, decl_cast_u16
decl_v2 1, decl_cast_u16
decl_v2 1, decl_ops U32
decl_v2 1, decl_cast_u32
decl_v2 1, decl_cast_f32
Reference in New Issue View Git Blame Copy Permalink