FFmpeg
ops_asmgen.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2026 Ramiro Polla
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "ops_asmgen.h"
22 
23 /*********************************************************************/
24 /* Helpers functions. */
25 
26 /* Looping when s->use_vh is set. */
27 #define LOOP_VH(s, mask, idx) if (s->use_vh) LOOP(mask, idx)
28 #define LOOP_MASK_VH(s, p, idx) if (s->use_vh) LOOP_MASK(p, idx)
29 #define LOOP_MASK_BWD_VH(s, p, idx) if (s->use_vh) LOOP_MASK_BWD(p, idx)
30 
31 /* Inline rasm comments. */
32 #define CMT(comment) rasm_annotate(r, comment)
33 #define CMTF(fmt, ...) rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__)
34 
35 /* Reshape input/output vector registers for current SwsOp. */
36 static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
37 {
38  for (int i = 0; i < 4; i++) {
39  regs->sl[i] = a64op_make_vec(a64op_vec_n(regs->sl[i]), el_count, el_size);
40  regs->sh[i] = a64op_make_vec(a64op_vec_n(regs->sh[i]), el_count, el_size);
41  regs->dl[i] = a64op_make_vec(a64op_vec_n(regs->dl[i]), el_count, el_size);
42  regs->dh[i] = a64op_make_vec(a64op_vec_n(regs->dh[i]), el_count, el_size);
43  }
44 }
45 
46 /* Reshape temp vector registers for current SwsOp. */
47 static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
48 {
49  for (int i = 0; i < FF_ARRAY_ELEMS(regs->vt); i++)
50  regs->vt[i] = a64op_make_vec(a64op_vec_n(regs->vt[i]), el_count, el_size);
51 }
52 
53 /* Reshape const vector registers for current SwsOp. */
54 static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
55 {
56  for (int i = 0; i < FF_ARRAY_ELEMS(regs->vk); i++)
57  regs->vk[i] = a64op_make_vec(a64op_vec_n(regs->vk[i]), el_count, el_size);
58 }
59 
60 /*********************************************************************/
61 /* Function frame */
62 
63 static unsigned clobbered_frame_size(unsigned n)
64 {
65  return ((n + 1) >> 1) * 16;
66 }
67 
68 static void asmgen_prologue(SwsAArch64Context *s, const RasmOp *regs, unsigned n)
69 {
70  RasmContext *r = s->rctx;
71  RasmOp sp = a64op_sp();
72  unsigned frame_size = clobbered_frame_size(n);
73  RasmOp sp_pre = a64op_pre(sp, -frame_size);
74 
75  rasm_add_comment(r, "prologue");
76  if (n == 0) {
77  /* no-op */
78  } else if (n == 1) {
79  i_str(r, regs[0], sp_pre);
80  } else {
81  i_stp(r, regs[0], regs[1], sp_pre);
82  for (unsigned i = 2; i + 1 < n; i += 2)
83  i_stp(r, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t)));
84  if (n & 1)
85  i_str(r, regs[n - 1], a64op_off(sp, (n - 1) * sizeof(uint64_t)));
86  }
87 }
88 
89 static void asmgen_epilogue(SwsAArch64Context *s, const RasmOp *regs, unsigned n)
90 {
91  RasmContext *r = s->rctx;
92  RasmOp sp = a64op_sp();
93  unsigned frame_size = clobbered_frame_size(n);
94  RasmOp sp_post = a64op_post(sp, frame_size);
95 
96  rasm_add_comment(r, "epilogue");
97  if (n == 0) {
98  /* no-op */
99  } else if (n == 1) {
100  i_ldr(r, regs[0], sp_post);
101  } else {
102  if (n & 1)
103  i_ldr(r, regs[n - 1], a64op_off(sp, (n - 1) * sizeof(uint64_t)));
104  for (unsigned i = (n & ~1u) - 2; i >= 2; i -= 2)
105  i_ldp(r, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t)));
106  i_ldp(r, regs[0], regs[1], sp_post);
107  }
108 }
109 
110 /*********************************************************************/
111 /* Callee-saved registers (r19-r28, fp, and lr). */
112 #define MAX_SAVED_REGS 12
113 
114 static void clobber_gpr(RasmOp regs[MAX_SAVED_REGS], unsigned *count,
115  RasmOp gpr)
116 {
117  const int n = a64op_gpr_n(gpr);
118  if (n >= 19 && n <= 30)
119  regs[(*count)++] = gpr;
120 }
121 
122 static unsigned clobbered_gprs(const SwsAArch64Context *s,
123  SwsCompMask imask, SwsCompMask omask,
124  RasmOp regs[MAX_SAVED_REGS])
125 {
126  unsigned count = 0;
127  clobber_gpr(regs, &count, a64op_lr());
128  LOOP(imask, i) {
129  clobber_gpr(regs, &count, s->in[i]);
130  clobber_gpr(regs, &count, s->in_bump[i]);
131  }
132  LOOP(omask, i) {
133  clobber_gpr(regs, &count, s->out[i]);
134  clobber_gpr(regs, &count, s->out_bump[i]);
135  }
136  return count;
137 }
138 
139 static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
140 {
141  RasmContext *r = s->rctx;
142 
143  /**
144  * The process function for aarch64 works similarly to the x86 backend.
145  * The description in x86/ops_include.asm mostly holds as well here.
146  */
147 
148  /* Function prologue */
149  RasmOp saved_regs[MAX_SAVED_REGS];
150  unsigned nsaved = clobbered_gprs(s, imask, omask, saved_regs);
151  if (nsaved)
152  asmgen_prologue(s, saved_regs, nsaved);
153 
154  /* Load values from exec. */
155  RasmOp exec_in[4];
156  RasmOp exec_in_bump[4];
157  RasmOp exec_out[4];
158  RasmOp exec_out_bump[4];
159  LOOP(imask, i) { exec_in [i] = a64op_off(s->exec, offsetof_exec_in + (i * sizeof(uint8_t *))); }
160  LOOP(imask, i) { exec_in_bump [i] = a64op_off(s->exec, offsetof_exec_in_bump + (i * sizeof(uint8_t *))); }
161  LOOP(omask, i) { exec_out [i] = a64op_off(s->exec, offsetof_exec_out + (i * sizeof(uint8_t *))); }
162  LOOP(omask, i) { exec_out_bump[i] = a64op_off(s->exec, offsetof_exec_out_bump + (i * sizeof(uint8_t *))); }
163  LOOP(imask, i) { i_ldr(r, s->in[i], exec_in [i]); CMTF("in[%u] = exec->in[%u];", i, i); }
164  LOOP(omask, i) { i_ldr(r, s->out[i], exec_out[i]); CMTF("out[%u] = exec->out[%u];", i, i); }
165  LOOP(imask, i) { i_ldr(r, s->in_bump[i], exec_in_bump[i]); CMTF("in_bump[%u] = exec->in_bump[%u];", i, i); }
166  LOOP(omask, i) { i_ldr(r, s->out_bump[i], exec_out_bump[i]); CMTF("out_bump[%u] = exec->out_bump[%u];", i, i); }
167 
168  /* Setup. */
169  s->setup = rasm_get_current_node(r);
170 
171  int first_row = rasm_new_label(r, NULL);
172  int next_row = rasm_new_label(r, NULL);
173  int next_block = rasm_new_label(r, NULL);
174 
175  /* Jump to first row (skips padding). */
176  i_b (r, rasm_op_label(first_row)); CMT("goto first_row;");
177 
178  /* Perform padding, preparing for next row. */
179  rasm_add_label(r, next_row); CMT("next_row:");
180  LOOP(imask, i) { i_add(r, s->in[i], s->in[i], s->in_bump[i]); CMTF("in[%u] += in_bump[%u];", i, i); }
181  LOOP(omask, i) { i_add(r, s->out[i], s->out[i], s->out_bump[i]); CMTF("out[%u] += out_bump[%u];", i, i); }
182 
183  /* First row (reset x). */
184  rasm_add_label(r, first_row); CMT("first_row:");
185  i_mov(r, s->bx, s->bx_start); CMT("bx = bx_start;");
186 
187  /* Main loop. */
188  rasm_add_label(r, next_block); CMT("next_block:");
189  s->loop = rasm_get_current_node(r);
190 
191  /* Perform horizontal loop. */
192  i_add(r, s->bx, s->bx, IMM(1)); CMT("bx += 1;");
193  i_cmp(r, s->bx, s->bx_end); CMT("if (bx != bx_end)");
194  i_bne(r, next_block); CMT(" goto next_block;");
195 
196  /* Perform vertical loop. */
197  i_add(r, s->y, s->y, IMM(1)); CMT("y += 1;");
198  i_cmp(r, s->y, s->y_end); CMT("if (y != y_end)");
199  i_bne(r, next_row); CMT(" goto next_row;");
200 
201  /* Function epilogue */
202  if (nsaved)
203  asmgen_epilogue(s, saved_regs, nsaved);
204 
205  i_ret(r);
206 }
207 
208 /*********************************************************************/
209 /* gather raw pixels from planes */
210 /* SWS_UOP_READ_BIT */
211 /* SWS_UOP_READ_NIBBLE */
212 /* SWS_UOP_READ_PACKED */
213 /* SWS_UOP_READ_PLANAR */
214 
215 static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
216  SwsAArch64OpRegs *regs)
217 {
218  RasmContext *r = s->rctx;
219  AArch64VecViews dl[1] = { a64op_vec_views(regs->dl[0]) };
220  AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]);
221  AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]);
222 
223  AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]);
224  RasmOp wtmp = a64op_w(s->tmp0);
225 
226  /* Note that shift_vec has negative values, so that using it with
227  * ushl actually performs a right shift. */
228  if (p->block_size == 16) {
229  i_ldrh(r, wtmp, a64op_post(s->in[0], 2)); CMT("uint16_t tmp = *in[0]++;");
230  i_dup (r, dl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);");
231  i_lsr (r, wtmp, wtmp, IMM(8)); CMT("tmp >>= 8;");
232  i_dup (r, vtmp.b8, wtmp); CMT("vtmp.lo = broadcast(tmp);");
233  i_ins (r, dl[0].de[1], vtmp.de[0]); CMT("vl[0].hi = vtmp.lo;");
234  i_ushl(r, dl[0].b16, dl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;");
235  i_and (r, dl[0].b16, dl[0].b16, bitmask_vec.b16); CMT("vl[0] &= bitmask_vec;");
236  } else {
237  i_ldrb(r, wtmp, a64op_post(s->in[0], 1)); CMT("uint8_t tmp = *in[0]++;");
238  i_dup (r, dl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);");
239  i_ushl(r, dl[0].b8, dl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;");
240  i_and (r, dl[0].b8, dl[0].b8, bitmask_vec.b8); CMT("vl[0] &= bitmask_vec;");
241  }
242 }
243 
244 static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
245  SwsAArch64OpRegs *regs)
246 {
247  RasmContext *r = s->rctx;
248  AArch64VecViews dl[1] = { a64op_vec_views(regs->dl[0]) };
249  AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]);
250 
251  AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]);
252 
253  if (p->block_size == 8) {
254  i_ldr (r, dl[0].s, a64op_post(s->in[0], 4)); CMT("vl[0] = *in[0]++;");
255  i_ushr(r, vtmp.b8, dl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;");
256  i_and (r, dl[0].b8, dl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;");
257  i_zip1(r, dl[0].b8, vtmp.b8, dl[0].b8); CMT("interleave");
258  } else {
259  i_ldr (r, dl[0].d, a64op_post(s->in[0], 8)); CMT("vl[0] = *in[0]++;");
260  i_ushr(r, vtmp.b8, dl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;");
261  i_and (r, dl[0].b8, dl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;");
262  i_zip1(r, dl[0].b16, vtmp.b16, dl[0].b16); CMT("interleave");
263  }
264 }
265 
266 static void asmgen_op_read_packed_n(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx)
267 {
268  RasmContext *r = s->rctx;
269 
270  switch (p->mask) {
271  case SWS_COMP_ELEMS(2): i_ld2(r, vv_2(vx[0], vx[1]), a64op_post(s->in[0], s->vec_size * 2)); break;
272  case SWS_COMP_ELEMS(3): i_ld3(r, vv_3(vx[0], vx[1], vx[2]), a64op_post(s->in[0], s->vec_size * 3)); break;
273  case SWS_COMP_ELEMS(4): i_ld4(r, vv_4(vx[0], vx[1], vx[2], vx[3]), a64op_post(s->in[0], s->vec_size * 4)); break;
274  }
275 }
276 
277 static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
278  SwsAArch64OpRegs *regs)
279 {
280  av_assert0(p->mask != 0x0001);
281  asmgen_op_read_packed_n(s, p, regs->dl);
282  if (s->use_vh)
283  asmgen_op_read_packed_n(s, p, regs->dh);
284 }
285 
286 static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
287  SwsAArch64OpRegs *regs)
288 {
289  RasmContext *r = s->rctx;
290  AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl);
291  AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh);
292 
293  LOOP_MASK(p, i) {
294  switch ((s->use_vh ? 0x100 : 0) | s->vec_size) {
295  case 0x008: i_ldr(r, dl[i].d, a64op_post(s->in[i], s->vec_size * 1)); break;
296  case 0x010: i_ldr(r, dl[i].q, a64op_post(s->in[i], s->vec_size * 1)); break;
297  case 0x108: i_ldp(r, dl[i].d, dh[i].d, a64op_post(s->in[i], s->vec_size * 2)); break;
298  case 0x110: i_ldp(r, dl[i].q, dh[i].q, a64op_post(s->in[i], s->vec_size * 2)); break;
299  }
300  }
301 }
302 
303 /*********************************************************************/
304 /* write raw pixels to planes */
305 /* SWS_UOP_WRITE_BIT */
306 /* SWS_UOP_WRITE_NIBBLE */
307 /* SWS_UOP_WRITE_PACKED */
308 /* SWS_UOP_WRITE_PLANAR */
309 
310 static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
311  SwsAArch64OpRegs *regs)
312 {
313  RasmContext *r = s->rctx;
314  AArch64VecViews sl[1] = { a64op_vec_views(regs->sl[0]) };
315  AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]);
316 
317  AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]);
318  AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]);
319 
320  if (p->block_size == 8) {
321  i_ushl(r, sl[0].b8, sl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;");
322  i_addv(r, vtmp0.b, sl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);");
323  i_str (r, vtmp0.b, a64op_post(s->out[0], 1)); CMT("*out[0]++ = vtmp0;");
324  } else {
325  i_ushl(r, sl[0].b16, sl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;");
326  i_addv(r, vtmp0.b, sl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);");
327  i_ins (r, vtmp1.de[0], sl[0].de[1]); CMT("vtmp1.lo = vl[0].hi;");
328  i_addv(r, vtmp1.b, vtmp1.b8); CMT("vtmp1[0] = add_across(vtmp1);");
329  i_ins (r, vtmp0.be[1], vtmp1.be[0]); CMT("vtmp0[1] = vtmp1[0];");
330  i_str (r, vtmp0.h, a64op_post(s->out[0], 2)); CMT("*out[0]++ = vtmp0;");
331  }
332 }
333 
334 static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
335  SwsAArch64OpRegs *regs)
336 {
337  RasmContext *r = s->rctx;
338  AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl);
339  AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]);
340  AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]);
341 
342  if (p->block_size == 8) {
343  i_shl (r, vtmp0.h4, sl[0].h4, IMM(4));
344  i_ushr(r, vtmp1.h4, sl[0].h4, IMM(8));
345  i_orr (r, sl[0].b8, vtmp0.b8, vtmp1.b8);
346  i_xtn (r, vtmp0.b8, sl[0].h8);
347  i_str (r, vtmp0.s, a64op_post(s->out[0], 4));
348  } else {
349  i_shl (r, vtmp0.h8, sl[0].h8, IMM(4));
350  i_ushr(r, vtmp1.h8, sl[0].h8, IMM(8));
351  i_orr (r, sl[0].b16, vtmp0.b16, vtmp1.b16);
352  i_xtn (r, vtmp0.b8, sl[0].h8);
353  i_str (r, vtmp0.d, a64op_post(s->out[0], 8));
354  }
355 }
356 
357 static void asmgen_op_write_packed_n(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx)
358 {
359  RasmContext *r = s->rctx;
360 
361  switch (p->mask) {
362  case SWS_COMP_ELEMS(2): i_st2(r, vv_2(vx[0], vx[1]), a64op_post(s->out[0], s->vec_size * 2)); break;
363  case SWS_COMP_ELEMS(3): i_st3(r, vv_3(vx[0], vx[1], vx[2]), a64op_post(s->out[0], s->vec_size * 3)); break;
364  case SWS_COMP_ELEMS(4): i_st4(r, vv_4(vx[0], vx[1], vx[2], vx[3]), a64op_post(s->out[0], s->vec_size * 4)); break;
365  }
366 }
367 
368 static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
369  SwsAArch64OpRegs *regs)
370 {
371  av_assert0(p->mask != 0x0001);
372  asmgen_op_write_packed_n(s, p, regs->sl);
373  if (s->use_vh)
374  asmgen_op_write_packed_n(s, p, regs->sh);
375 }
376 
377 static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
378  SwsAArch64OpRegs *regs)
379 {
380  RasmContext *r = s->rctx;
381  AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl);
382  AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh);
383 
384  LOOP_MASK(p, i) {
385  switch ((s->use_vh ? 0x100 : 0) | s->vec_size) {
386  case 0x008: i_str(r, sl[i].d, a64op_post(s->out[i], s->vec_size * 1)); break;
387  case 0x010: i_str(r, sl[i].q, a64op_post(s->out[i], s->vec_size * 1)); break;
388  case 0x108: i_stp(r, sl[i].d, sh[i].d, a64op_post(s->out[i], s->vec_size * 2)); break;
389  case 0x110: i_stp(r, sl[i].q, sh[i].q, a64op_post(s->out[i], s->vec_size * 2)); break;
390  }
391  }
392 }
393 
394 /*********************************************************************/
395 /* swap byte order (for differing endianness) */
396 /* SWS_UOP_SWAP_BYTES */
397 
398 static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
399  SwsAArch64OpRegs *regs)
400 {
401  RasmContext *r = s->rctx;
402  AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl);
403  AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh);
404  AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl);
405  AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh);
406 
407  switch (ff_sws_pixel_type_size(p->type)) {
408  case sizeof(uint16_t):
409  LOOP_MASK (p, i) i_rev16(r, dl[i].b16, sl[i].b16);
410  LOOP_MASK_VH(s, p, i) i_rev16(r, dh[i].b16, sh[i].b16);
411  break;
412  case sizeof(uint32_t):
413  LOOP_MASK (p, i) i_rev32(r, dl[i].b16, sl[i].b16);
414  LOOP_MASK_VH(s, p, i) i_rev32(r, dh[i].b16, sh[i].b16);
415  break;
416  }
417 }
418 
419 /*********************************************************************/
420 /* rearrange channel order, or duplicate channels */
421 /* SWS_UOP_PERMUTE */
422 /* SWS_UOP_COPY */
423 
424 static const char *print_swizzle_v(char buf[8], int8_t n, uint8_t vh)
425 {
426  if (n == -1)
427  snprintf(buf, sizeof(char[8]), "vtmp%c", vh ? 'h' : 'l');
428  else
429  snprintf(buf, sizeof(char[8]), "v%c[%u]", vh ? 'h' : 'l', n);
430  return buf;
431 }
432 #define PRINT_SWIZZLE_V(n, vh) print_swizzle_v((char[8]){ 0 }, n, vh)
433 
434 static RasmOp swizzle_a64op(SwsAArch64OpRegs *regs, int8_t n, uint8_t vh, bool dst)
435 {
436  if (n == -1)
437  return regs->vt[vh];
438  if (vh)
439  return dst ? regs->dh[n] : regs->sh[n];
440  return dst ? regs->dl[n] : regs->sl[n];
441 }
442 
443 static void swizzle_emit(SwsAArch64Context *s, SwsAArch64OpRegs *regs,
444  int8_t dst, int8_t src)
445 {
446  RasmContext *r = s->rctx;
447  RasmOp src_op[2] = { swizzle_a64op(regs, src, 0, false), swizzle_a64op(regs, src, 1, false) };
448  RasmOp dst_op[2] = { swizzle_a64op(regs, dst, 0, true), swizzle_a64op(regs, dst, 1, true) };
449 
450  i_mov (r, dst_op[0], src_op[0]); CMTF("%s = %s;", PRINT_SWIZZLE_V(dst, 0), PRINT_SWIZZLE_V(src, 0));
451  if (s->use_vh) {
452  i_mov(r, dst_op[1], src_op[1]); CMTF("%s = %s;", PRINT_SWIZZLE_V(dst, 1), PRINT_SWIZZLE_V(src, 1));
453  }
454 }
455 
456 static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
457  SwsAArch64OpRegs *regs)
458 {
459  for (int i = 0; i < p->par.move.num_moves; i++)
460  swizzle_emit(s, regs, p->par.move.dst[i], p->par.move.src[i]);
461 }
462 
463 /*********************************************************************/
464 /* split tightly packed data into components */
465 /* SWS_UOP_UNPACK */
466 
467 static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
468  SwsAArch64OpRegs *regs)
469 {
470  RasmContext *r = s->rctx;
471  RasmOp *sl = regs->sl;
472  RasmOp *sh = regs->sh;
473  RasmOp *dl = regs->dl;
474  RasmOp *dh = regs->dh;
475  RasmOp *vmask = regs->vk;
476 
477  const int offsets[4] = {
478  p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1],
479  p->par.pack.pattern[3] + p->par.pack.pattern[2],
480  p->par.pack.pattern[3],
481  0
482  };
483 
484  /* Loop backwards to avoid clobbering component 0. */
485  LOOP_MASK_BWD (p, i) {
486  if (offsets[i]) {
487  i_ushr (r, dl[i], sl[0], IMM(offsets[i])); CMTF("vl[%u] >>= %u;", i, offsets[i]);
488  } else if (i) {
489  i_mov16b(r, dl[i], sl[0]); CMTF("vl[%u] = vl[0];", i);
490  }
491  }
492  LOOP_MASK_BWD_VH(s, p, i) {
493  if (offsets[i]) {
494  i_ushr (r, dh[i], sh[0], IMM(offsets[i])); CMTF("vh[%u] >>= %u;", i, offsets[i]);
495  } else if (i) {
496  i_mov16b(r, dh[i], sh[0]); CMTF("vh[%u] = vh[0];", i);
497  }
498  }
499 
500  /* Apply masks. */
501  LOOP_MASK_BWD (p, i) { i_and16b(r, dl[i], dl[i], vmask[i]); CMTF("vl[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); }
502  LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, dh[i], dh[i], vmask[i]); CMTF("vh[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); }
503 }
504 
505 /*********************************************************************/
506 /* compress components into tightly packed data */
507 /* SWS_UOP_PACK */
508 
509 static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
510  SwsAArch64OpRegs *regs)
511 {
512  RasmContext *r = s->rctx;
513  RasmOp *sl = regs->sl;
514  RasmOp *sh = regs->sh;
515  RasmOp *dl = regs->dl;
516  RasmOp *dh = regs->dh;
517 
518  const int offsets[4] = {
519  p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1],
520  p->par.pack.pattern[3] + p->par.pack.pattern[2],
521  p->par.pack.pattern[3],
522  0
523  };
524  SwsCompMask offset_mask = 0;
525  LOOP_MASK(p, i) {
526  if (offsets[i])
527  offset_mask |= SWS_COMP(i);
528  }
529 
530  /* Perform left shift. */
531  LOOP (offset_mask, i) { i_shl(r, dl[i], sl[i], IMM(offsets[i])); CMTF("vl[%u] <<= %u;", i, offsets[i]); }
532  LOOP_VH(s, offset_mask, i) { i_shl(r, dh[i], sh[i], IMM(offsets[i])); CMTF("vh[%u] <<= %u;", i, offsets[i]); }
533  LOOP (offset_mask, i) { sl[i] = dl[i]; }
534  LOOP_VH(s, offset_mask, i) { sh[i] = dh[i]; }
535 
536  /* Combine components. */
537  for (int i = 0; i < 4; i++) {
538  sl[i] = v_16b(sl[i]);
539  sh[i] = v_16b(sh[i]);
540  dl[i] = v_16b(dl[i]);
541  dh[i] = v_16b(dh[i]);
542  }
543  LOOP_MASK (p, i) {
544  if (i != 0) {
545  i_orr16b (r, dl[0], sl[0], sl[i]); CMTF("vl[0] |= vl[%u];", i);
546  if (s->use_vh) {
547  i_orr16b(r, dh[0], sh[0], sh[i]); CMTF("vh[0] |= vh[%u];", i);
548  }
549  }
550  }
551 }
552 
553 /*********************************************************************/
554 /* logical left shift of raw pixel values */
555 /* SWS_UOP_LSHIFT */
556 
557 static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
558  SwsAArch64OpRegs *regs)
559 {
560  uint8_t shift = p->par.shift.amount;
561  RasmContext *r = s->rctx;
562  RasmOp *sl = regs->sl;
563  RasmOp *sh = regs->sh;
564  RasmOp *dl = regs->dl;
565  RasmOp *dh = regs->dh;
566 
567  LOOP_MASK (p, i) { i_shl(r, dl[i], sl[i], IMM(shift)); CMTF("vl[%u] <<= %u;", i, shift); }
568  LOOP_MASK_VH(s, p, i) { i_shl(r, dh[i], sh[i], IMM(shift)); CMTF("vh[%u] <<= %u;", i, shift); }
569 }
570 
571 /*********************************************************************/
572 /* right shift of raw pixel values */
573 /* SWS_UOP_RSHIFT */
574 
575 static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
576  SwsAArch64OpRegs *regs)
577 {
578  uint8_t shift = p->par.shift.amount;
579  RasmContext *r = s->rctx;
580  RasmOp *sl = regs->sl;
581  RasmOp *sh = regs->sh;
582  RasmOp *dl = regs->dl;
583  RasmOp *dh = regs->dh;
584 
585  LOOP_MASK (p, i) { i_ushr(r, dl[i], sl[i], IMM(shift)); CMTF("vl[%u] >>= %u;", i, shift); }
586  LOOP_MASK_VH(s, p, i) { i_ushr(r, dh[i], sh[i], IMM(shift)); CMTF("vh[%u] >>= %u;", i, shift); }
587 }
588 
589 /*********************************************************************/
590 /* clear pixel values */
591 /* SWS_UOP_CLEAR */
592 
593 static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
594  RasmOp *vx, RasmOp *vk, int i, const char *vx_str)
595 {
596  RasmContext *r = s->rctx;
597  RasmOp clear_vec = vk[0];
598  if (p->par.clear.zero & SWS_COMP(i)) {
599  i_movi(r, vx[i], IMM(0)); CMTF("%s[%u] = 0;", vx_str, i);
600  } else if (p->par.clear.one & SWS_COMP(i)) {
601  if (p->block_size * ff_sws_pixel_type_size(p->type) == 8) {
602  i_movi(r, v_8b (vx[i]), IMM(0xff));
603  } else {
604  i_movi(r, v_16b(vx[i]), IMM(0xff));
605  }
606  CMTF("%s[%u] = UINT_MAX;", vx_str, i);
607  } else {
608  i_dup (r, vx[i], a64op_elem(clear_vec, i)); CMTF("%s[%u] = broadcast(clear_vec[%u]);", vx_str, i, i);
609  }
610 }
611 
612 static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
613  SwsAArch64OpRegs *regs)
614 {
615  RasmOp *dl = regs->dl;
616  RasmOp *dh = regs->dh;
617  RasmOp *vk = regs->vk;
618 
619  LOOP_MASK (p, i) { emit_clear(s, p, dl, vk, i, "vl"); }
620  LOOP_MASK_VH(s, p, i) { emit_clear(s, p, dh, vk, i, "vh"); }
621 }
622 
623 /*********************************************************************/
624 /* convert (cast) between formats */
625 /* SWS_UOP_TO_U8 */
626 /* SWS_UOP_TO_U16 */
627 /* SWS_UOP_TO_U32 */
628 /* SWS_UOP_TO_F32 */
629 
630 static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
631  SwsAArch64OpRegs *regs)
632 {
633  RasmContext *r = s->rctx;
634  AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl);
635  AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh);
636  AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl);
637  AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh);
638 
639  /**
640  * Since each instruction in the convert operation needs specific
641  * element types, it is simpler to use arrangement specifiers for
642  * each operand instead of reshaping all vectors.
643  */
644 
645  size_t src_el_size = s->el_size;
646  SwsPixelType to_type;
647  switch (p->uop) {
648  case SWS_UOP_TO_U8: to_type = SWS_PIXEL_U8; break;
649  case SWS_UOP_TO_U16: to_type = SWS_PIXEL_U16; break;
650  case SWS_UOP_TO_U32: to_type = SWS_PIXEL_U32; break;
651  case SWS_UOP_TO_F32: to_type = SWS_PIXEL_F32; break;
652  default:
653  av_assert0(!"Invalid uop!");
654  break;
655  }
656  size_t dst_el_size = ff_sws_pixel_type_size(to_type);
657 
658  /**
659  * This function assumes block_size is either 8 or 16, and that
660  * we're always using the most amount of vector registers possible.
661  * Therefore, u32 always uses the high vector bank.
662  */
663  if (p->type == SWS_PIXEL_F32) {
664  rasm_add_comment(r, "f32 -> u32");
665  LOOP_MASK(p, i) i_fcvtzu(r, dl[i].s4, sl[i].s4);
666  LOOP_MASK(p, i) i_fcvtzu(r, dh[i].s4, sh[i].s4);
667  memcpy(sl, dl, sizeof(sl));
668  memcpy(sh, dh, sizeof(sh));
669  }
670 
671  if (p->block_size == 8) {
672  if (src_el_size == 1 && dst_el_size > src_el_size) {
673  rasm_add_comment(r, "u8 -> u16");
674  LOOP_MASK(p, i) i_uxtl (r, dl[i].h8, sl[i].b8);
675  memcpy(sl, dl, sizeof(sl));
676  src_el_size = 2;
677  } else if (src_el_size == 4 && dst_el_size < src_el_size) {
678  rasm_add_comment(r, "u32 -> u16");
679  LOOP_MASK(p, i) i_xtn (r, dl[i].h4, sl[i].s4);
680  LOOP_MASK(p, i) i_xtn (r, dh[i].h4, sh[i].s4);
681  LOOP_MASK(p, i) i_ins (r, dl[i].de[1], sh[i].de[0]);
682  memcpy(sl, dl, sizeof(sl));
683  memcpy(sh, dh, sizeof(sh));
684  src_el_size = 2;
685  }
686  if (src_el_size == 2 && dst_el_size == 4) {
687  rasm_add_comment(r, "u16 -> u32");
688  LOOP_MASK(p, i) i_uxtl2(r, dh[i].s4, sl[i].h8);
689  LOOP_MASK(p, i) i_uxtl (r, dl[i].s4, sl[i].h4);
690  memcpy(sl, dl, sizeof(sl));
691  memcpy(sh, dh, sizeof(sh));
692  src_el_size = 4;
693  } else if (src_el_size == 2 && dst_el_size == 1) {
694  rasm_add_comment(r, "u16 -> u8");
695  LOOP_MASK(p, i) i_xtn (r, dl[i].b8, sl[i].h8);
696  memcpy(sl, dl, sizeof(sl));
697  src_el_size = 1;
698  }
699  } else /* if (p->block_size == 16) */ {
700  if (src_el_size == 1 && dst_el_size == 2) {
701  rasm_add_comment(r, "u8 -> u16");
702  LOOP_MASK(p, i) i_uxtl2(r, dh[i].h8, sl[i].b16);
703  LOOP_MASK(p, i) i_uxtl (r, dl[i].h8, sl[i].b8);
704  memcpy(sl, dl, sizeof(sl));
705  memcpy(sh, dh, sizeof(sh));
706  } else if (src_el_size == 2 && dst_el_size == 1) {
707  rasm_add_comment(r, "u16 -> u8");
708  LOOP_MASK(p, i) i_xtn (r, dl[i].b8, sl[i].h8);
709  LOOP_MASK(p, i) i_xtn (r, dh[i].b8, sh[i].h8);
710  LOOP_MASK(p, i) i_ins (r, dl[i].de[1], sh[i].de[0]);
711  memcpy(sl, dl, sizeof(sl));
712  }
713  }
714 
715  /* See comment above for high vector bank usage for u32. */
716  if (to_type == SWS_PIXEL_F32) {
717  rasm_add_comment(r, "u32 -> f32");
718  LOOP_MASK(p, i) i_ucvtf(r, dl[i].s4, sl[i].s4);
719  LOOP_MASK(p, i) i_ucvtf(r, dh[i].s4, sh[i].s4);
720  }
721 }
722 
723 /*********************************************************************/
724 /* expand integers to the full range */
725 /* SWS_UOP_EXPAND_PAIR */
726 /* SWS_UOP_EXPAND_QUAD */
727 
728 static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
729  SwsAArch64OpRegs *regs)
730 {
731  RasmContext *r = s->rctx;
732  RasmOp *sl = regs->sl;
733  RasmOp *dl = regs->dl;
734  RasmOp *dh = regs->dh;
735 
736  size_t src_el_size = s->el_size;
737  SwsPixelType to_type;
738  switch (p->uop) {
739  case SWS_UOP_EXPAND_PAIR: to_type = SWS_PIXEL_U16; break;
740  case SWS_UOP_EXPAND_QUAD: to_type = SWS_PIXEL_U32; break;
741  default:
742  av_assert0(!"Invalid uop!");
743  break;
744  }
745  size_t dst_el_size = ff_sws_pixel_type_size(to_type);
746  size_t dst_total_size = p->block_size * dst_el_size;
747  size_t dst_vec_size = FFMIN(dst_total_size, 16);
748 
749  if (!s->use_vh)
750  s->use_vh = (dst_vec_size != dst_total_size);
751 
752  if (src_el_size == 1) {
753  rasm_add_comment(r, "u8 -> u16");
754  reshape_io_vectors(regs, 16, 1);
755  LOOP_MASK_VH(s, p, i) i_zip2(r, dh[i], sl[i], sl[i]);
756  LOOP_MASK (p, i) i_zip1(r, dl[i], sl[i], sl[i]);
757  sl = dl;
758  }
759  if (dst_el_size == 4) {
760  rasm_add_comment(r, "u16 -> u32");
761  reshape_io_vectors(regs, 8, 2);
762  LOOP_MASK_VH(s, p, i) i_zip2(r, dh[i], sl[i], sl[i]);
763  LOOP_MASK (p, i) i_zip1(r, dl[i], sl[i], sl[i]);
764  }
765 }
766 
767 /*********************************************************************/
768 /* numeric minimum */
769 /* SWS_UOP_MIN */
770 
771 static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
772  SwsAArch64OpRegs *regs)
773 {
774  RasmContext *r = s->rctx;
775  RasmOp *sl = regs->sl;
776  RasmOp *sh = regs->sh;
777  RasmOp *dl = regs->dl;
778  RasmOp *dh = regs->dh;
779  RasmOp *vk = regs->vk;
780 
781  if (p->type == SWS_PIXEL_F32) {
782  LOOP_MASK (p, i) { i_fmin(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); }
783  LOOP_MASK_VH(s, p, i) { i_fmin(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); }
784  } else {
785  LOOP_MASK (p, i) { i_umin(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); }
786  LOOP_MASK_VH(s, p, i) { i_umin(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); }
787  }
788 }
789 
790 /*********************************************************************/
791 /* numeric maximum */
792 /* SWS_UOP_MAX */
793 
794 static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
795  SwsAArch64OpRegs *regs)
796 {
797  RasmContext *r = s->rctx;
798  RasmOp *sl = regs->sl;
799  RasmOp *sh = regs->sh;
800  RasmOp *dl = regs->dl;
801  RasmOp *dh = regs->dh;
802  RasmOp *vk = regs->vk;
803 
804  if (p->type == SWS_PIXEL_F32) {
805  LOOP_MASK (p, i) { i_fmax(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); }
806  LOOP_MASK_VH(s, p, i) { i_fmax(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); }
807  } else {
808  LOOP_MASK (p, i) { i_umax(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); }
809  LOOP_MASK_VH(s, p, i) { i_umax(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); }
810  }
811 }
812 
813 /*********************************************************************/
814 /* multiplication by scalar */
815 /* SWS_UOP_SCALE */
816 
817 static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
818  SwsAArch64OpRegs *regs)
819 {
820  RasmContext *r = s->rctx;
821  RasmOp *sl = regs->sl;
822  RasmOp *sh = regs->sh;
823  RasmOp *dl = regs->dl;
824  RasmOp *dh = regs->dh;
825  RasmOp scale_vec = regs->vk[0];
826 
827  if (p->type == SWS_PIXEL_F32) {
828  LOOP_MASK (p, i) { i_fmul(r, dl[i], sl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); }
829  LOOP_MASK_VH(s, p, i) { i_fmul(r, dh[i], sh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); }
830  } else {
831  LOOP_MASK (p, i) { i_mul (r, dl[i], sl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); }
832  LOOP_MASK_VH(s, p, i) { i_mul (r, dh[i], sh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); }
833  }
834 }
835 
836 /*********************************************************************/
837 /* generalized linear affine transform */
838 /* SWS_UOP_LINEAR */
839 /* SWS_UOP_LINEAR_FMA */
840 
841 static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
842  SwsAArch64OpRegs *regs, bool vh_pass)
843 {
844  RasmContext *r = s->rctx;
845  /**
846  * The intermediate registers for fmul+fadd (for when SWS_BITEXACT
847  * is set) start from temp vector 8.
848  */
849  RasmOp *vt = regs->vt;
850  RasmOp *vtmp = &vt[8];
851  RasmOp *sx = vh_pass ? regs->sh : regs->sl;
852  RasmOp *dx = vh_pass ? regs->dh : regs->dl;
853  char cvh = vh_pass ? 'h' : 'l';
854 
855  if (vh_pass && !s->use_vh)
856  return;
857 
858  /**
859  * The non-zero coefficients have been packed in aarch64_setup_linear()
860  * in sequential order into the individual lanes of the coefficient
861  * vector registers. We must follow the same order of execution here.
862  */
863  LOOP_MASK(p, i) {
864  bool first = true;
865  RasmNode *pre_mul = rasm_get_current_node(r);
866  for (int j = 0; j < 5; j++) {
867  bool is_offset = (j == 0);
868  int src_j = is_offset ? 4 : (j - 1);
869  if (p->par.lin.zero & SWS_MASK(i, src_j))
870  continue;
871  RasmOp vsrc = sx[src_j];
872  RasmOp vcoeff = regs->linear_vcoeff[i][j];
873  if (first && is_offset) {
874  i_dup (r, dx[i], vcoeff); CMTF("v%c[%u] = broadcast(offset[%u]);", cvh, i, i);
875  } else if (first && !is_offset) {
876  if (p->par.lin.one & SWS_MASK(i, src_j)) {
877  i_mov16b(r, dx[i], vsrc); CMTF("v%c[%u] = vsrc%c[%u];", cvh, i, cvh, src_j);
878  } else {
879  i_fmul (r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] = vsrc%c[%u] * coeff[%u][%u];", cvh, i, cvh, src_j, i, src_j);
880  }
881  } else if (p->uop == SWS_UOP_LINEAR_FMA) {
882  /**
883  * Most modern aarch64 cores have a fastpath for sequences
884  * of fmla instructions. This means that even if the coefficient
885  * is 1, it is still faster to use fmla by 1 instead of fadd.
886  */
887  i_fmla(r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] += vsrc%c[%u] * coeff[%u][%u];", cvh, i, cvh, src_j, i, src_j);
888  } else {
889  /**
890  * Split the multiply-accumulate into fmul+fadd. All
891  * multiplications are performed first into temporary
892  * registers, and only then added to the destination,
893  * to reduce the dependency chain.
894  * There is no need to perform multiplications by 1.
895  */
896  if (!(p->par.lin.one & SWS_MASK(i, src_j))) {
897  pre_mul = rasm_set_current_node(r, pre_mul);
898  i_fmul(r, vtmp[src_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * coeff[%u][%u];", src_j, cvh, src_j, i, src_j);
899  pre_mul = rasm_set_current_node(r, pre_mul);
900  i_fadd(r, dx[i], dx[i], vtmp[src_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, src_j);
901  } else {
902  i_fadd(r, dx[i], dx[i], vsrc); CMTF("v%c[%u] += vsrc%c[%u];", cvh, i, cvh, src_j);
903  }
904  }
905  first = false;
906  }
907  }
908 }
909 
910 static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
911  SwsAArch64OpRegs *regs)
912 {
913  /* Perform linear passes for low and high vector banks. */
914  linear_pass(s, p, regs, false);
915  linear_pass(s, p, regs, true);
916 }
917 
918 /*********************************************************************/
919 /* add dithering noise */
920 /* SWS_UOP_DITHER */
921 
922 static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p,
923  SwsAArch64OpRegs *regs)
924 {
925  RasmContext *r = s->rctx;
926  RasmOp *sl = regs->sl;
927  RasmOp *sh = regs->sh;
928  RasmOp *dl = regs->dl;
929  RasmOp *dh = regs->dh;
930  RasmOp src_ptr = regs->dither_ptr;
931 
932  RasmOp ptr = s->tmp0;
933  RasmOp tmp1 = s->tmp1;
934  RasmOp wtmp1 = a64op_w(tmp1);
935  RasmOp dither_vl = regs->vt[0];
936  RasmOp dither_vh = regs->vt[1];
937  RasmOp bx64 = a64op_x(s->bx);
938  RasmOp y64 = a64op_x(s->y);
939 
940  /**
941  * For a description of the matrix buffer layout, read the comments
942  * in aarch64_setup_dither() in aarch64/ops.c.
943  */
944 
945  /**
946  * Sort components by y_offset value so that we can start dithering
947  * with the smallest value, and increment the pointer upwards for
948  * each new offset. The dither matrix is over-allocated and may be
949  * over-read at the top, but it cannot be over-read before the start
950  * of the buffer. Since we only mask the y offset once, this would
951  * be an issue if we tried to subtract a value larger than the
952  * initial y_offset.
953  */
954  int sorted[4];
955  int n_comps = 0;
956  /* Very cheap bucket sort. */
957  int max_offset = 0;
958  LOOP_MASK(p, i)
959  max_offset = FFMAX(max_offset, p->par.dither.y_offset[i]);
960  for (int y_off = 0; y_off <= max_offset; y_off++) {
961  LOOP_MASK(p, i) {
962  if (p->par.dither.y_offset[i] == y_off)
963  sorted[n_comps++] = i;
964  }
965  }
966 
967  /**
968  * We use ubfiz to mask and shift left in one single instruction:
969  * ubfiz <Wd>, <Wn>, #<lsb>, #<width>
970  * Wd = (Wn & ((1 << width) - 1)) << lsb;
971  *
972  * Given:
973  * block_size = 8, log2(block_size) = 3
974  * dither_size = 16, log2(dither_size) = 4, dither_mask = 0b1111
975  * sizeof(float) = 4, log2(sizeof(float)) = 2
976  *
977  * Suppose we have bx = 0bvvvv. To get x, we left shift by
978  * log2(block_size) and end up with 0bvvvv000. Then we mask against
979  * dither_mask, and end up with 0bv000. Finally we multiply by
980  * sizeof(float), which is the same as shifting left by
981  * log2(sizeof(float)). The result is 0bv00000.
982  *
983  * Therefore:
984  * width = log2(dither_size) - log2(block_size)
985  * lsb = log2(block_size) + log2(sizeof(float))
986  */
987  const int block_size_log2 = (p->block_size == 16) ? 4 : 3;
988  const int dither_size_log2 = p->par.dither.size_log2;
989  const int sizeof_float_log2 = 2;
990  if (dither_size_log2 != block_size_log2) {
991  RasmOp lsb = IMM(block_size_log2 + sizeof_float_log2);
992  RasmOp width = IMM(dither_size_log2 - block_size_log2);
993  i_ubfiz(r, tmp1, bx64, lsb, width); CMT("tmp1 = (bx & ((dither_size / block_size) - 1)) * block_size * sizeof(float);");
994  i_add (r, ptr, src_ptr, tmp1); CMT("ptr += tmp1;");
995  src_ptr = ptr;
996  }
997 
998  int last_y_off = -1;
999  int prev_i = 0;
1000  for (int sorted_i = 0; sorted_i < n_comps; sorted_i++) {
1001  int i = sorted[sorted_i];
1002  uint8_t y_off = p->par.dither.y_offset[i];
1003  bool do_load = (y_off != last_y_off);
1004 
1005  if (last_y_off < 0) {
1006  /* On the first run, calculate pointer inside dither_matrix. */
1007  RasmOp lsb = IMM(dither_size_log2 + sizeof_float_log2);
1008  RasmOp width = IMM(dither_size_log2);
1009  /**
1010  * The ubfiz instruction for the y offset performs masking
1011  * by the dither matrix size and shifts by the stride.
1012  */
1013  if (y_off == 0) {
1014  i_ubfiz(r, tmp1, y64, lsb, width); CMT("tmp1 = (y & (dither_size - 1)) * dither_size * sizeof(float);");
1015  } else {
1016  i_add (r, wtmp1, s->y, IMM(y_off)); CMTF("tmp1 = y + y_off[%u];", i);
1017  i_ubfiz(r, tmp1, tmp1, lsb, width); CMT("tmp1 = (tmp1 & (dither_size - 1)) * dither_size * sizeof(float);");
1018  }
1019  i_add(r, ptr, src_ptr, tmp1); CMT("ptr += tmp1;");
1020  } else if (do_load) {
1021  /**
1022  * On subsequent runs, just increment the pointer.
1023  * The matrix is over-allocated, so we don't risk
1024  * overreading.
1025  */
1026  int delta = (y_off - last_y_off) * (1 << dither_size_log2) * sizeof(float);
1027  i_add(r, ptr, ptr, IMM(delta)); CMTF("ptr += (y_off[%u] - y_off[%u]) * dither_size * sizeof(float);", i, prev_i);
1028  }
1029 
1030  if (do_load) {
1031  RasmOp dither_vlq = v_q(dither_vl);
1032  RasmOp dither_vhq = v_q(dither_vh);
1033  i_ldp (r, dither_vlq, dither_vhq, a64op_base(ptr)); CMT("{ ditherl, ditherh } = *ptr;");
1034  }
1035 
1036  i_fadd (r, dl[i], sl[i], dither_vl); CMTF("vl[%u] += vditherl;", i);
1037  if (s->use_vh) {
1038  i_fadd(r, dh[i], sh[i], dither_vh); CMTF("vh[%u] += vditherh;", i);
1039  }
1040  sl = dl;
1041  sh = dh;
1042 
1043  last_y_off = y_off;
1044  prev_i = i;
1045  }
1046 }
AArch64VecViews::h8
RasmOp h8
Definition: rasm.h:467
r
const char * r
Definition: vf_curves.c:127
LOOP_MASK_BWD_VH
#define LOOP_MASK_BWD_VH(s, p, idx)
Definition: ops_asmgen.c:29
LOOP_MASK_BWD
#define LOOP_MASK_BWD(p, idx)
Definition: ops_impl.h:64
a64op_base
static RasmOp a64op_base(RasmOp op)
Definition: rasm.h:506
AArch64VecViews::b16
RasmOp b16
Definition: rasm.h:465
i_zip1
#define i_zip1(rctx, op0, op1, op2)
Definition: rasm.h:604
AArch64VecViews::d
RasmOp d
Definition: rasm.h:461
i_ld4
#define i_ld4(rctx, op0, op1)
Definition: rasm.h:569
i_mul
#define i_mul(rctx, op0, op1, op2)
Definition: rasm.h:577
a64op_w
static RasmOp a64op_w(RasmOp op)
Definition: rasm.h:360
clobber_gpr
static void clobber_gpr(RasmOp regs[MAX_SAVED_REGS], unsigned *count, RasmOp gpr)
Definition: ops_asmgen.c:114
rasm_set_current_node
RasmNode * rasm_set_current_node(RasmContext *rctx, RasmNode *node)
Definition: rasm.c:199
i_st4
#define i_st4(rctx, op0, op1)
Definition: rasm.h:586
u
#define u(width, name, range_min, range_max)
Definition: cbs_apv.c:68
AArch64VecViews
This helper structure is used to mimic the assembler syntax for vector register modifiers.
Definition: rasm.h:456
rasm_get_current_node
RasmNode * rasm_get_current_node(RasmContext *rctx)
Definition: rasm.c:194
i_ld3
#define i_ld3(rctx, op0, op1)
Definition: rasm.h:568
SwsAArch64OpRegs::vk
RasmOp vk[4]
Definition: ops_asmgen.h:33
vv_2
static RasmOp vv_2(RasmOp op0, RasmOp op1)
Definition: rasm.h:447
asmgen_op_read_bit
static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:215
vv_3
static RasmOp vv_3(RasmOp op0, RasmOp op1, RasmOp op2)
Definition: rasm.h:448
SWS_UOP_LINEAR_FMA
@ SWS_UOP_LINEAR_FMA
Definition: uops.h:176
CMTF
#define CMTF(fmt,...)
Definition: ops_asmgen.c:33
PRINT_SWIZZLE_V
#define PRINT_SWIZZLE_V(n, vh)
Definition: ops_asmgen.c:432
reshape_temp_vectors
static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition: ops_asmgen.c:47
i_dup
#define i_dup(rctx, op0, op1)
Definition: rasm.h:557
i_bne
#define i_bne(rctx, id)
Definition: rasm.h:609
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
A64OP_VEC_VIEWS4
#define A64OP_VEC_VIEWS4(op)
Definition: rasm.h:479
i_ld2
#define i_ld2(rctx, op0, op1)
Definition: rasm.h:567
i_fmla
#define i_fmla(rctx, op0, op1, op2)
Definition: rasm.h:562
RasmNode
Definition: rasm.h:145
i_rev16
#define i_rev16(rctx, op0, op1)
Definition: rasm.h:580
IMM
#define IMM(val)
Definition: rasm.h:92
asmgen_op_write_nibble
static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:334
reshape_const_vectors
static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition: ops_asmgen.c:54
print_swizzle_v
static const char * print_swizzle_v(char buf[8], int8_t n, uint8_t vh)
Definition: ops_asmgen.c:424
i_fmin
#define i_fmin(rctx, op0, op1, op2)
Definition: rasm.h:561
a64op_vec_n
static uint8_t a64op_vec_n(RasmOp op)
Definition: rasm.h:377
LOOP_MASK_VH
#define LOOP_MASK_VH(s, p, idx)
Definition: ops_asmgen.c:28
SwsAArch64Context
Definition: ops_asmgen.h:43
asmgen_op_write_bit
static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:310
AArch64VecViews::s
RasmOp s
Definition: rasm.h:460
i_st2
#define i_st2(rctx, op0, op1)
Definition: rasm.h:584
i_st3
#define i_st3(rctx, op0, op1)
Definition: rasm.h:585
i_ushr
#define i_ushr(rctx, op0, op1, op2)
Definition: rasm.h:600
RasmOp
Runtime assembler for AArch64.
Definition: rasm.h:44
SWS_UOP_TO_U16
@ SWS_UOP_TO_U16
Definition: uops.h:159
SwsAArch64OpRegs::dither_ptr
RasmOp dither_ptr
Definition: ops_asmgen.h:37
i_addv
#define i_addv(rctx, op0, op1)
Definition: rasm.h:548
asmgen_op_convert
static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:630
SwsAArch64OpRegs::vt
RasmOp vt[12]
Definition: ops_asmgen.h:32
rasm_add_label
RasmNode * rasm_add_label(RasmContext *rctx, int id)
Definition: rasm.c:146
SWS_COMP_ELEMS
#define SWS_COMP_ELEMS(N)
Definition: uops.h:99
i_fadd
#define i_fadd(rctx, op0, op1, op2)
Definition: rasm.h:558
first
trying all byte sequences megabyte in length and selecting the best looking sequence will yield cases to try But first
Definition: rate_distortion.txt:12
a64op_elem
static RasmOp a64op_elem(RasmOp op, uint8_t idx)
Definition: rasm.h:422
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
asmgen_op_lshift
static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:557
SwsAArch64OpRegs::sl
RasmOp sl[4]
Definition: ops_asmgen.h:28
float
float
Definition: af_crystalizer.c:122
AArch64VecViews::be
RasmOp be[2]
Definition: rasm.h:472
AArch64VecViews::b
RasmOp b
Definition: rasm.h:458
offsets
static const int offsets[]
Definition: hevc_pel.c:34
LOOP
#define LOOP(mask, idx)
Definition: ops_impl.h:56
frame_size
int frame_size
Definition: mxfenc.c:2489
i_orr16b
#define i_orr16b(rctx, op0, op1, op2)
Definition: rasm.h:628
ops_asmgen.h
SWS_UOP_TO_F32
@ SWS_UOP_TO_F32
Definition: uops.h:161
CMT
#define CMT(comment)
Definition: ops_asmgen.c:32
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:42
asmgen_op_rshift
static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:575
SwsCompMask
uint8_t SwsCompMask
Bit-mask of components.
Definition: uops.h:87
asmgen_op_swap_bytes
static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:398
i_ins
#define i_ins(rctx, op0, op1)
Definition: rasm.h:564
v_8b
static RasmOp v_8b(RasmOp op)
Definition: rasm.h:437
i_ldr
#define i_ldr(rctx, op0, op1)
Definition: rasm.h:571
reshape_io_vectors
static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition: ops_asmgen.c:36
a64op_make_vec
static RasmOp a64op_make_vec(uint8_t n, uint8_t el_count, uint8_t el_size)
Definition: rasm.h:366
ff_sws_pixel_type_size
static av_const int ff_sws_pixel_type_size(SwsPixelType type)
Definition: uops.h:49
asmgen_op_write_packed
static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:368
i_cmp
#define i_cmp(rctx, op0, op1)
Definition: rasm.h:555
offsetof_exec_out_bump
#define offsetof_exec_out_bump
Definition: ops_impl.h:84
NULL
#define NULL
Definition: coverity.c:32
asmgen_op_read_planar
static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:286
i_fmul
#define i_fmul(rctx, op0, op1, op2)
Definition: rasm.h:563
asmgen_op_write_packed_n
static void asmgen_op_write_packed_n(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx)
Definition: ops_asmgen.c:357
a64op_post
static RasmOp a64op_post(RasmOp op, int16_t imm)
Definition: rasm.h:509
i_umin
#define i_umin(rctx, op0, op1, op2)
Definition: rasm.h:595
MAX_SAVED_REGS
#define MAX_SAVED_REGS
Definition: ops_asmgen.c:112
LOOP_VH
#define LOOP_VH(s, mask, idx)
Definition: ops_asmgen.c:27
offsetof_exec_out
#define offsetof_exec_out
Definition: ops_impl.h:82
SwsAArch64OpRegs::dh
RasmOp dh[4]
Definition: ops_asmgen.h:31
asmgen_op_clear
static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:612
SwsPixelType
SwsPixelType
Definition: uops.h:38
i_add
#define i_add(rctx, op0, op1, op2)
Definition: rasm.h:547
SWS_UOP_TO_U32
@ SWS_UOP_TO_U32
Definition: uops.h:160
rasm_new_label
int rasm_new_label(RasmContext *rctx, const char *name)
Allocate a new label ID with the given name.
Definition: rasm.c:282
asmgen_op_pack
static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:509
a64op_sp
static RasmOp a64op_sp(void)
Definition: rasm.h:357
i_uxtl
#define i_uxtl(rctx, op0, op1)
Definition: rasm.h:601
linear_pass
static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs, bool vh_pass)
Definition: ops_asmgen.c:841
asmgen_op_min
static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:771
SwsAArch64OpRegs::sh
RasmOp sh[4]
Definition: ops_asmgen.h:29
LOOP_MASK
#define LOOP_MASK(p, idx)
Definition: ops_impl.h:63
shift
static int shift(int a, int b)
Definition: bonk.c:261
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:87
clobbered_gprs
static unsigned clobbered_gprs(const SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask, RasmOp regs[MAX_SAVED_REGS])
Definition: ops_asmgen.c:122
i
#define i(width, name, range_min, range_max)
Definition: cbs_h264.c:63
i_ldrb
#define i_ldrb(rctx, op0, op1)
Definition: rasm.h:572
i_shl
#define i_shl(rctx, op0, op1, op2)
Definition: rasm.h:582
i_fmax
#define i_fmax(rctx, op0, op1, op2)
Definition: rasm.h:560
AArch64VecViews::h
RasmOp h
Definition: rasm.h:459
i_zip2
#define i_zip2(rctx, op0, op1, op2)
Definition: rasm.h:605
SWS_COMP
#define SWS_COMP(X)
Definition: uops.h:96
i_fcvtzu
#define i_fcvtzu(rctx, op0, op1)
Definition: rasm.h:559
SWS_PIXEL_U32
@ SWS_PIXEL_U32
Definition: uops.h:42
asmgen_op_read_packed_n
static void asmgen_op_read_packed_n(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx)
Definition: ops_asmgen.c:266
i_ucvtf
#define i_ucvtf(rctx, op0, op1)
Definition: rasm.h:593
a64op_off
static RasmOp a64op_off(RasmOp op, int16_t imm)
Definition: rasm.h:507
asmgen_op_dither
static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:922
i_uxtl2
#define i_uxtl2(rctx, op0, op1)
Definition: rasm.h:602
a64op_vec_views
AArch64VecViews a64op_vec_views(RasmOp op)
Definition: rasm.c:330
AArch64VecViews::de
RasmOp de[2]
Definition: rasm.h:473
asmgen_op_move
static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:456
asmgen_op_max
static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:794
vv_4
static RasmOp vv_4(RasmOp op0, RasmOp op1, RasmOp op2, RasmOp op3)
Definition: rasm.h:449
i_lsr
#define i_lsr(rctx, op0, op1, op2)
Definition: rasm.h:574
clobbered_frame_size
static unsigned clobbered_frame_size(unsigned n)
Definition: ops_asmgen.c:63
RasmContext
Definition: rasm.h:185
SWS_UOP_TO_U8
@ SWS_UOP_TO_U8
Definition: uops.h:158
i_ldp
#define i_ldp(rctx, op0, op1, op2)
Definition: rasm.h:570
swizzle_emit
static void swizzle_emit(SwsAArch64Context *s, SwsAArch64OpRegs *regs, int8_t dst, int8_t src)
Definition: ops_asmgen.c:443
SWS_PIXEL_U8
@ SWS_PIXEL_U8
Definition: uops.h:40
s
uint8_t s
Definition: llvidencdsp.c:39
delta
float delta
Definition: vorbis_enc_data.h:430
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
SwsAArch64OpRegs::linear_vcoeff
RasmOp linear_vcoeff[4][5]
Definition: ops_asmgen.h:38
AArch64VecViews::h4
RasmOp h4
Definition: rasm.h:466
asmgen_op_unpack
static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:467
swizzle_a64op
static RasmOp swizzle_a64op(SwsAArch64OpRegs *regs, int8_t n, uint8_t vh, bool dst)
Definition: ops_asmgen.c:434
i_xtn
#define i_xtn(rctx, op0, op1)
Definition: rasm.h:603
asmgen_op_scale
static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:817
a64op_pre
static RasmOp a64op_pre(RasmOp op, int16_t imm)
Definition: rasm.h:508
asmgen_op_expand
static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:728
rasm_op_label
static RasmOp rasm_op_label(int id)
Definition: rasm.h:96
i_umax
#define i_umax(rctx, op0, op1, op2)
Definition: rasm.h:594
offsetof_exec_in
#define offsetof_exec_in
These values will be used by ops_asmgen to access fields inside of SwsOpExec and SwsOpImpl.
Definition: ops_impl.h:81
SwsAArch64OpRegs::dl
RasmOp dl[4]
Definition: ops_asmgen.h:30
i_mov16b
#define i_mov16b(rctx, op0, op1)
Definition: rasm.h:627
i_b
#define i_b(rctx, op0)
Definition: rasm.h:551
i_str
#define i_str(rctx, op0, op1)
Definition: rasm.h:588
asmgen_op_linear
static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:910
i_and
#define i_and(rctx, op0, op1, op2)
Definition: rasm.h:550
i_ldrh
#define i_ldrh(rctx, op0, op1)
Definition: rasm.h:573
asmgen_prologue
static void asmgen_prologue(SwsAArch64Context *s, const RasmOp *regs, unsigned n)
Definition: ops_asmgen.c:68
asmgen_process
static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
Definition: ops_asmgen.c:139
v_16b
static RasmOp v_16b(RasmOp op)
Definition: rasm.h:438
i_ubfiz
#define i_ubfiz(rctx, op0, op1, op2, op3)
Definition: rasm.h:592
Windows::Graphics::DirectX::Direct3D11::p
IDirect3DDxgiInterfaceAccess _COM_Outptr_ void ** p
Definition: vsrc_gfxcapture_winrt.hpp:53
asmgen_epilogue
static void asmgen_epilogue(SwsAArch64Context *s, const RasmOp *regs, unsigned n)
Definition: ops_asmgen.c:89
i_orr
#define i_orr(rctx, op0, op1, op2)
Definition: rasm.h:578
SwsAArch64OpRegs
Definition: ops_asmgen.h:27
SwsAArch64OpImplParams
SwsAArch64OpImplParams describes the parameters for an SwsUOpType operation.
Definition: ops_impl.h:47
i_rev32
#define i_rev32(rctx, op0, op1)
Definition: rasm.h:581
i_stp
#define i_stp(rctx, op0, op1, op2)
Definition: rasm.h:587
SWS_PIXEL_F32
@ SWS_PIXEL_F32
Definition: uops.h:43
i_movi
#define i_movi(rctx, op0, op1)
Definition: rasm.h:576
emit_clear
static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx, RasmOp *vk, int i, const char *vx_str)
Definition: ops_asmgen.c:593
SWS_UOP_EXPAND_QUAD
@ SWS_UOP_EXPAND_QUAD
Definition: uops.h:157
i_ret
#define i_ret(rctx)
Definition: rasm.h:579
a64op_x
static RasmOp a64op_x(RasmOp op)
Definition: rasm.h:361
offsetof_exec_in_bump
#define offsetof_exec_in_bump
Definition: ops_impl.h:83
i_and16b
#define i_and16b(rctx, op0, op1, op2)
Definition: rasm.h:626
asmgen_op_write_planar
static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:377
nibble_mask
static uint16_t nibble_mask(SwsCompMask mask)
Definition: ops_impl.h:34
width
#define width
Definition: dsp.h:89
asmgen_op_read_nibble
static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:244
SWS_PIXEL_U16
@ SWS_PIXEL_U16
Definition: uops.h:41
SWS_MASK
#define SWS_MASK(I, J)
Definition: uops.h:229
snprintf
#define snprintf
Definition: snprintf.h:34
i_ushl
#define i_ushl(rctx, op0, op1, op2)
Definition: rasm.h:597
rasm_add_comment
RasmNode * rasm_add_comment(RasmContext *rctx, const char *comment)
Definition: rasm.c:117
src
#define src
Definition: vp8dsp.c:248
SWS_UOP_EXPAND_PAIR
@ SWS_UOP_EXPAND_PAIR
Definition: uops.h:156
a64op_lr
static RasmOp a64op_lr(void)
Definition: rasm.h:356
a64op_gpr_n
static uint8_t a64op_gpr_n(RasmOp op)
Definition: rasm.h:351
i_mov
#define i_mov(rctx, op0, op1)
Definition: rasm.h:575
v_q
static RasmOp v_q(RasmOp op)
Definition: rasm.h:434
AArch64VecViews::b8
RasmOp b8
Definition: rasm.h:464
asmgen_op_read_packed
static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition: ops_asmgen.c:277
Generated on Sun Jul 26 2026 19:23:51 for FFmpeg by   doxygen 1.8.17