#include "ops_asmgen.h"

Macros
#define	LOOP_VH(s, mask, idx) if (s->use_vh) LOOP(mask, idx)

#define	LOOP_MASK_VH(s, p, idx) if (s->use_vh) LOOP_MASK(p, idx)

#define	LOOP_MASK_BWD_VH(s, p, idx) if (s->use_vh) LOOP_MASK_BWD(p, idx)

#define	CMT(comment) rasm_annotate(r, comment)

#define	CMTF(fmt, ...) rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__)

#define	MAX_SAVED_REGS 12

#define	PRINT_SWIZZLE_V(n, vh) print_swizzle_v((char[8]){ 0 }, n, vh)

Functions
static void	reshape_io_vectors (SwsAArch64OpRegs *regs, int el_count, int el_size)

static void	reshape_temp_vectors (SwsAArch64OpRegs *regs, int el_count, int el_size)

static void	reshape_const_vectors (SwsAArch64OpRegs *regs, int el_count, int el_size)

static unsigned	clobbered_frame_size (unsigned n)

static void	asmgen_prologue (SwsAArch64Context s, const RasmOp regs, unsigned n)

static void	asmgen_epilogue (SwsAArch64Context s, const RasmOp regs, unsigned n)

static void	clobber_gpr (RasmOp regs[MAX_SAVED_REGS], unsigned *count, RasmOp gpr)

static unsigned	clobbered_gprs (const SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask, RasmOp regs[MAX_SAVED_REGS])

static void	asmgen_process (SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)

static void	asmgen_op_read_bit (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_read_nibble (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_read_packed_n (SwsAArch64Context s, const SwsAArch64OpImplParams p, RasmOp *vx)

static void	asmgen_op_read_packed (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_read_planar (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_write_bit (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_write_nibble (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_write_packed_n (SwsAArch64Context s, const SwsAArch64OpImplParams p, RasmOp *vx)

static void	asmgen_op_write_packed (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_write_planar (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_swap_bytes (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static const char *	print_swizzle_v (char buf[8], int8_t n, uint8_t vh)

static RasmOp	swizzle_a64op (SwsAArch64OpRegs *regs, int8_t n, uint8_t vh, bool dst)

static void	swizzle_emit (SwsAArch64Context s, SwsAArch64OpRegs regs, int8_t dst, int8_t src)

static void	asmgen_op_move (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_unpack (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_pack (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_lshift (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_rshift (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	emit_clear (SwsAArch64Context s, const SwsAArch64OpImplParams p, RasmOp vx, RasmOp vk, int i, const char *vx_str)

static void	asmgen_op_clear (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_convert (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_expand (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_min (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_max (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_scale (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	linear_pass (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs, bool vh_pass)

static void	asmgen_op_linear (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

static void	asmgen_op_dither (SwsAArch64Context s, const SwsAArch64OpImplParams p, SwsAArch64OpRegs *regs)

Macro Definition Documentation

◆ LOOP_VH

#define LOOP_VH	(	s,
		mask,
		idx
	)	if (s->use_vh) LOOP(mask, idx)

Definition at line 27 of file ops_asmgen.c.

◆ LOOP_MASK_VH

#define LOOP_MASK_VH	(	s,
		p,
		idx
	)	if (s->use_vh) LOOP_MASK(p, idx)

Definition at line 28 of file ops_asmgen.c.

◆ LOOP_MASK_BWD_VH

#define LOOP_MASK_BWD_VH	(	s,
		p,
		idx
	)	if (s->use_vh) LOOP_MASK_BWD(p, idx)

Definition at line 29 of file ops_asmgen.c.

◆ CMT

#define CMT ( comment ) rasm_annotate(r, comment)

Definition at line 32 of file ops_asmgen.c.

◆ CMTF

#define CMTF	(	fmt,
		...
	)	rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__)

Definition at line 33 of file ops_asmgen.c.

◆ MAX_SAVED_REGS

#define MAX_SAVED_REGS 12

Definition at line 112 of file ops_asmgen.c.

◆ PRINT_SWIZZLE_V

#define PRINT_SWIZZLE_V	(	n,
		vh
	)	print_swizzle_v((char[8]){ 0 }, n, vh)

Definition at line 432 of file ops_asmgen.c.

Function Documentation

◆ reshape_io_vectors()

static void reshape_io_vectors	(	SwsAArch64OpRegs *	regs,
		int	el_count,
		int	el_size
	)

static

Definition at line 36 of file ops_asmgen.c.

Referenced by asmgen_op_cps(), and asmgen_op_expand().

◆ reshape_temp_vectors()

static void reshape_temp_vectors	(	SwsAArch64OpRegs *	regs,
		int	el_count,
		int	el_size
	)

static

Definition at line 47 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ reshape_const_vectors()

static void reshape_const_vectors	(	SwsAArch64OpRegs *	regs,
		int	el_count,
		int	el_size
	)

static

Definition at line 54 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ clobbered_frame_size()

static unsigned clobbered_frame_size ( unsigned n )

static

Definition at line 63 of file ops_asmgen.c.

Referenced by asmgen_epilogue(), and asmgen_prologue().

◆ asmgen_prologue()

static void asmgen_prologue	(	SwsAArch64Context *	s,
		const RasmOp *	regs,
		unsigned	n
	)

static

Definition at line 68 of file ops_asmgen.c.

Referenced by asmgen_process().

◆ asmgen_epilogue()

static void asmgen_epilogue	(	SwsAArch64Context *	s,
		const RasmOp *	regs,
		unsigned	n
	)

static

Definition at line 89 of file ops_asmgen.c.

Referenced by asmgen_process().

◆ clobber_gpr()

static void clobber_gpr	(	RasmOp	regs[MAX_SAVED_REGS],
		unsigned *	count,
		RasmOp	gpr
	)

static

Definition at line 114 of file ops_asmgen.c.

Referenced by clobbered_gprs().

◆ clobbered_gprs()

static unsigned clobbered_gprs	(	const SwsAArch64Context *	s,
		SwsCompMask	imask,
		SwsCompMask	omask,
		RasmOp	regs[MAX_SAVED_REGS]
	)

static

Definition at line 122 of file ops_asmgen.c.

Referenced by asmgen_process().

◆ asmgen_process()

static void asmgen_process	(	SwsAArch64Context *	s,
		SwsCompMask	imask,
		SwsCompMask	omask
	)

static

The process function for aarch64 works similarly to the x86 backend. The description in x86/ops_include.asm mostly holds as well here.

Definition at line 139 of file ops_asmgen.c.

Referenced by asmgen_process_cps().

◆ asmgen_op_read_bit()

static void asmgen_op_read_bit	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 215 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_read_nibble()

static void asmgen_op_read_nibble	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 244 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_read_packed_n()

static void asmgen_op_read_packed_n	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		RasmOp *	vx
	)

static

Definition at line 266 of file ops_asmgen.c.

Referenced by asmgen_op_read_packed().

◆ asmgen_op_read_packed()

static void asmgen_op_read_packed	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 277 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_read_planar()

static void asmgen_op_read_planar	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 286 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_write_bit()

static void asmgen_op_write_bit	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 310 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_write_nibble()

static void asmgen_op_write_nibble	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 334 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_write_packed_n()

static void asmgen_op_write_packed_n	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		RasmOp *	vx
	)

static

Definition at line 357 of file ops_asmgen.c.

Referenced by asmgen_op_write_packed().

◆ asmgen_op_write_packed()

static void asmgen_op_write_packed	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 368 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_write_planar()

static void asmgen_op_write_planar	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 377 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_swap_bytes()

static void asmgen_op_swap_bytes	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 398 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ print_swizzle_v()

static const char* print_swizzle_v	(	char	buf[8],
		int8_t	n,
		uint8_t	vh
	)

static

Definition at line 424 of file ops_asmgen.c.

◆ swizzle_a64op()

static RasmOp swizzle_a64op	(	SwsAArch64OpRegs *	regs,
		int8_t	n,
		uint8_t	vh,
		bool	dst
	)

static

Definition at line 434 of file ops_asmgen.c.

Referenced by swizzle_emit().

◆ swizzle_emit()

static void swizzle_emit	(	SwsAArch64Context *	s,
		SwsAArch64OpRegs *	regs,
		int8_t	dst,
		int8_t	src
	)

static

Definition at line 443 of file ops_asmgen.c.

Referenced by asmgen_op_move().

◆ asmgen_op_move()

static void asmgen_op_move	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 456 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_unpack()

static void asmgen_op_unpack	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 467 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_pack()

static void asmgen_op_pack	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 509 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_lshift()

static void asmgen_op_lshift	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 557 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_rshift()

static void asmgen_op_rshift	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 575 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ emit_clear()

static void emit_clear	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		RasmOp *	vx,
		RasmOp *	vk,
		int	i,
		const char *	vx_str
	)

static

Definition at line 593 of file ops_asmgen.c.

Referenced by asmgen_op_clear().

◆ asmgen_op_clear()

static void asmgen_op_clear	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 612 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_convert()

static void asmgen_op_convert	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Since each instruction in the convert operation needs specific element types, it is simpler to use arrangement specifiers for each operand instead of reshaping all vectors.

This function assumes block_size is either 8 or 16, and that we're always using the most amount of vector registers possible. Therefore, u32 always uses the high vector bank.

Definition at line 630 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_expand()

static void asmgen_op_expand	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 728 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_min()

static void asmgen_op_min	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 771 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_max()

static void asmgen_op_max	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 794 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_scale()

static void asmgen_op_scale	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 817 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ linear_pass()

static void linear_pass	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs,
		bool	vh_pass
	)

static

The intermediate registers for fmul+fadd (for when SWS_BITEXACT is set) start from temp vector 8.

The non-zero coefficients have been packed in aarch64_setup_linear() in sequential order into the individual lanes of the coefficient vector registers. We must follow the same order of execution here.

Most modern aarch64 cores have a fastpath for sequences of fmla instructions. This means that even if the coefficient is 1, it is still faster to use fmla by 1 instead of fadd.

Split the multiply-accumulate into fmul+fadd. All multiplications are performed first into temporary registers, and only then added to the destination, to reduce the dependency chain. There is no need to perform multiplications by 1.

Definition at line 841 of file ops_asmgen.c.

Referenced by asmgen_op_linear().

◆ asmgen_op_linear()

static void asmgen_op_linear	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

Definition at line 910 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

◆ asmgen_op_dither()

static void asmgen_op_dither	(	SwsAArch64Context *	s,
		const SwsAArch64OpImplParams *	p,
		SwsAArch64OpRegs *	regs
	)

static

For a description of the matrix buffer layout, read the comments in aarch64_setup_dither() in aarch64/ops.c.

Sort components by y_offset value so that we can start dithering with the smallest value, and increment the pointer upwards for each new offset. The dither matrix is over-allocated and may be over-read at the top, but it cannot be over-read before the start of the buffer. Since we only mask the y offset once, this would be an issue if we tried to subtract a value larger than the initial y_offset.

We use ubfiz to mask and shift left in one single instruction: ubfiz <Wd>, <Wn>, #<lsb>, #<width> Wd = (Wn & ((1 << width) - 1)) << lsb;

Given: block_size = 8, log2(block_size) = 3 dither_size = 16, log2(dither_size) = 4, dither_mask = 0b1111 sizeof(float) = 4, log2(sizeof(float)) = 2

Suppose we have bx = 0bvvvv. To get x, we left shift by log2(block_size) and end up with 0bvvvv000. Then we mask against dither_mask, and end up with 0bv000. Finally we multiply by sizeof(float), which is the same as shifting left by log2(sizeof(float)). The result is 0bv00000.

Therefore: width = log2(dither_size) - log2(block_size) lsb = log2(block_size) + log2(sizeof(float))

The ubfiz instruction for the y offset performs masking by the dither matrix size and shifts by the stride.

On subsequent runs, just increment the pointer. The matrix is over-allocated, so we don't risk overreading.

Definition at line 922 of file ops_asmgen.c.

Referenced by asmgen_op_cps().

Macros

Functions

Macro Definition Documentation

◆ LOOP_VH

◆ LOOP_MASK_VH

◆ LOOP_MASK_BWD_VH

◆ CMT

◆ CMTF

◆ MAX_SAVED_REGS

◆ PRINT_SWIZZLE_V

Function Documentation

◆ reshape_io_vectors()

◆ reshape_temp_vectors()

◆ reshape_const_vectors()

◆ clobbered_frame_size()

◆ asmgen_prologue()

◆ asmgen_epilogue()

◆ clobber_gpr()

◆ clobbered_gprs()

◆ asmgen_process()

◆ asmgen_op_read_bit()

◆ asmgen_op_read_nibble()

◆ asmgen_op_read_packed_n()

◆ asmgen_op_read_packed()

◆ asmgen_op_read_planar()

◆ asmgen_op_write_bit()

◆ asmgen_op_write_nibble()

◆ asmgen_op_write_packed_n()

◆ asmgen_op_write_packed()

◆ asmgen_op_write_planar()

◆ asmgen_op_swap_bytes()

◆ print_swizzle_v()

◆ swizzle_a64op()

◆ swizzle_emit()

◆ asmgen_op_move()

◆ asmgen_op_unpack()

◆ asmgen_op_pack()

◆ asmgen_op_lshift()

◆ asmgen_op_rshift()

◆ emit_clear()

◆ asmgen_op_clear()

◆ asmgen_op_convert()

◆ asmgen_op_expand()

◆ asmgen_op_min()

◆ asmgen_op_max()

◆ asmgen_op_scale()

◆ linear_pass()

◆ asmgen_op_linear()

◆ asmgen_op_dither()