psx_gpu: add a tool to generate asm offsets
[pcsx_rearmed.git] / plugins / gpu_neon / psx_gpu / psx_gpu_arm_neon.S
1 /*
2  * Copyright (C) 2011 Gilead Kutnick "Exophase" <exophase@gmail.com>
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of
7  * the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  */
14
15 #define MAX_SPANS                                         512
16 #define MAX_BLOCKS                                        64
17 #define MAX_BLOCKS_PER_ROW                                128
18
19 #include "psx_gpu_offsets.h"
20
21 #define psx_gpu_b_dx_offset (psx_gpu_b_block_span_offset + 4)
22
23 #define edge_data_left_x_offset                           0
24 #define edge_data_num_blocks_offset                       2
25 #define edge_data_right_mask_offset                       4
26 #define edge_data_y_offset                                6
27
28
29 #define psx_gpu                                           r0
30 #define v_a                                               r1
31 #define v_b                                               r2
32 #define v_c                                               r3
33
34 #define x0                                                r4
35 #define x1                                                r5
36 #define x2                                                r6
37 #define x0_x1                                             r5
38 #define x1_x2                                             r6
39 #define y0                                                r7
40 #define y1                                                r8
41 #define y2                                                r9
42 #define y0_y1                                             r7
43 #define y1_y2                                             r8
44 #define b0                                                r9
45 #define b1                                                r10
46 #define b2                                                r11
47 #define b0_b1                                             r10
48 #define b1_b2                                             r11
49
50
51 #define area_r_s                                          r5
52
53 #define g_bx0                                             r2
54 #define g_bx                                              r3
55 #define g_bx2                                             r4
56 #define g_bx3                                             r5
57 #define b_base                                            r6
58 #define g_by                                              r8
59
60 #define gs_bx                                             r7
61 #define gs_by                                             r10
62
63 #define ga_bx                                             g_bx
64 #define ga_by                                             g_by
65
66 #define gw_bx_h                                           g_bx
67 #define gw_by_h                                           g_by
68
69 #define gw_bx_l                                           r11
70 #define gw_by_l                                           gw_bx_l
71
72 #define store_a                                           r0
73 #define store_b                                           r1
74 #define store_inc                                         r5
75
76
77 #define v0                                                q0
78 #define uvrgb0                                            d0
79 #define x0_y0                                             d1
80
81 #define v1                                                q1
82 #define uvrgb1                                            d2
83 #define x1_y1                                             d3
84
85 #define v2                                                q2
86 #define uvrgb2                                            d4
87 #define x2_y2                                             d5
88
89 #define x0_ab                                             q3
90 #define uvrg_xxxx0                                        q3
91 #define uvrg0                                             d6
92 #define xxxx0                                             d7
93
94 #define x1_ab                                             q4
95 #define uvrg_xxxx1                                        q4
96 #define uvrg1                                             d8
97 #define xxxx1                                             d9
98
99 #define x2_ab                                             q5
100 #define uvrg_xxxx2                                        q5
101 #define uvrg2                                             d10
102 #define xxxx2                                             d11
103
104 #define y0_ab                                             q6
105 #define yyyy_uvrg0                                        q6
106 #define yyyy0                                             d12
107 #define uvrg0b                                            d13
108
109 #define y1_ab                                             q7
110 #define yyyy_uvrg1                                        q7
111 #define yyyy1                                             d14
112 #define uvrg1b                                            d15
113
114 #define y2_ab                                             q8
115 #define yyyy_uvrg2                                        q8
116 #define yyyy2                                             d16
117 #define uvrg2b                                            d17
118
119 #define d0_ab                                             q9
120 #define d0_a                                              d18
121 #define d0_b                                              d19
122
123 #define d1_ab                                             q10
124 #define d1_a                                              d20
125 #define d1_b                                              d21
126
127 #define d2_ab                                             q11
128 #define d2_a                                              d22
129 #define d2_b                                              d23
130
131 #define d3_ab                                             q12
132 #define d3_a                                              d24
133 #define d3_b                                              d25
134
135 #define ga_uvrg_x                                         q1
136 #define ga_uvrg_y                                         q4
137
138 #define dx                                                x0_x1
139 #define dy                                                y0_y1
140 #define db                                                b0_b1
141
142 #define uvrg_base                                         q11
143
144 #define gs_uvrg_x                                         q5
145 #define gs_uvrg_y                                         q6
146
147 #define g_uvrg_x                                          q1
148 #define ga_uv_x                                           d2
149 #define g_uv_x                                            d2
150 #define ga_rg_x                                           d3
151 #define g_rg_x                                            d3
152
153 #define g_uvrg_y                                          q4
154 #define ga_uv_y                                           d8
155 #define g_uv_y                                            d8
156 #define ga_rg_y                                           d9
157 #define g_rg_y                                            d9
158
159 #define gw_uv_x                                           q1
160 #define gw_rg_x                                           q2
161 #define gw_uv_y                                           q4
162 #define gw_rg_y                                           q3
163
164 #define w_mask                                            q9
165 #define w_mask_l                                          d18
166
167 #define r_shift                                           q10
168
169 #define uvrg_dx0                                          q0
170 #define uvrg_dx0l                                         d0
171 #define uvrg_dx0h                                         d1
172
173 #define uvrg_dx1                                          q1
174 #define uvrg_dx1l                                         d2
175 #define uvrg_dx1h                                         d3
176
177 #define uvrg_dx2                                          q2
178 #define uvrg_dx2l                                         d4
179 #define uvrg_dx2h                                         d5
180
181 #define uvrg_dx3                                          q3
182 #define uvrg_dx3l                                         d6
183 #define uvrg_dx3h                                         d7
184
185
186 .align 4
187
188 /* FIXME: users of this should be in psx_gpu instead */
189 #ifndef __PIC__
190 #define load_pointer(register, pointer)                                        \
191   movw register, :lower16:pointer;                                             \
192   movt register, :upper16:pointer;                                             \
193
194 #else
195 #define load_pointer(register, pointer)                                        \
196   ldr  register, =pointer                                                      \
197
198 #endif
199
200 #define function(name)                                                         \
201   .global name;                                                                \
202   name:                                                                        \
203
204 @ r0: psx_gpu
205 @ r1: v_a
206 @ r2: v_b
207 @ r3: v_c
208
209 function(compute_all_gradients)
210   // First compute the triangle area reciprocal and shift. The division will
211   // happen concurrently with much of the work which follows.
212   @ r12 = psx_gpu->triangle_area
213   ldr r12, [ psx_gpu, #psx_gpu_triangle_area_offset ]
214   stmdb sp!, { r4 - r11, lr }
215
216   @ load exponent of 62 into upper half of double
217   movw r4, #0
218   clz r14, r12                       @ r14 = shift
219
220   movt r4, #((62 + 1023) << 4)
221   mov r12, r12, lsl r14              @ r12 = triangle_area_normalized
222
223   @ load area normalized into lower half of double
224   mov r5, r12, lsr #10
225   vmov.f64 d30, r5, r4               @ d30 = (1 << 62) + ta_n
226
227   movt r4, #((1022 + 31) << 4)
228   mov r5, r12, lsl #20
229
230   add r4, r4, r12, lsr #11
231   vmov.f64 d31, r5, r4
232
233   vdiv.f64 d30, d30, d31             @ d30 = ((1 << 62) + ta_n) / ta_n
234
235   // ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0)) =
236   // ( d0       *  d1      ) - ( d2       *  d3      ) =
237   // ( m0                  ) - ( m1                  ) = gradient
238
239   // This is split to do 12 elements at a time over three sets: a, b, and c.
240   // Technically we only need to do 10 elements (uvrgb_x and uvrgb_y), so
241   // two of the slots are unused.
242
243   // Inputs are all 16-bit signed. The m0/m1 results are 32-bit signed, as
244   // is g.
245
246   // First type is:  uvrg bxxx xxxx 
247   // Second type is: yyyy ybyy uvrg 
248   // Since x_a and y_c are the same the same variable is used for both. 
249
250   vld1.u32 { v0 }, [ v_a, : 128 ]    @ v0 = { uvrg0, b0, x0, y0 }
251   ldrsh x0, [ v_a, #8 ]              @ load x0
252
253   vld1.u32 { v1 }, [ v_b, : 128 ]    @ v1 = { uvrg1, b1, x1, y1}
254   ldrh x1, [ v_b, #8 ]               @ load x1
255
256   vld1.u32 { v2 }, [ v_c, : 128 ]    @ v2 = { uvrg2, b2, x2, y2 }
257   ldrh x2, [ v_c, #8 ]               @ load x2
258
259   vmovl.u8 uvrg_xxxx0, uvrgb0        @ uvrg_xxxx0 = { uv0, rg0, b0-, -- }
260   ldrh y0, [ v_a, #10 ]              @ load y0
261
262   vmovl.u8 uvrg_xxxx1, uvrgb1        @ uvrg_xxxx1 = { uv1, rg1, b1-, -- }
263   ldrh y1, [ v_b, #10 ]              @ load y1
264
265   vmovl.u8 uvrg_xxxx2, uvrgb2        @ uvrg_xxxx2 = { uv2, rg2, b2-, -- }
266   ldrh y2, [ v_c, #10 ]              @ load y2
267
268   vmov.u8 uvrg0b, uvrg0              @ uvrg0b = { uv0, rg0 }
269   vdup.u16 xxxx0, x0_y0[0]           @ xxxx0 = { xx0, xx0 }
270
271   orr x1_x2, x1, x2, lsl #16         @ x1_x2 = { x1, x2 }
272   pkhbt x0_x1, x0, x1, lsl #16       @ x0_x1 = { x0, x1 }
273
274   vmov.u8 uvrg1b, uvrg1              @ uvrg1b = { uv1, rg1 }
275   vdup.u16 xxxx1, x1_y1[0]           @ xxxx1 = { xx1, xx1 }
276
277   vmov.u8 uvrg2b, uvrg2              @ uvrg2b = { uv2, rg2 }
278   vdup.u16 xxxx2, x2_y2[0]           @ xxxx2 = { xx2, xx2 }
279
280   ldrb b2, [ v_c, #4 ]               @ load b2
281   orr y0_y1, y0, y1, lsl #16         @ y0_y1 = { y0, y1 }
282
283   ldrb b1, [ v_b, #4 ]               @ load b1
284   orr y1_y2, y1, y2, lsl #16         @ y1_y2 = { y1, y2 }
285
286   vdup.u16 yyyy0, x0_y0[1]           @ yyyy0 = { yy0, yy0 }
287   vsub.s16 d0_ab, x1_ab, x0_ab
288
289   ldrb b0, [ v_a, #4 ]               @ load b0
290   orr b1_b2, b1, b2, lsl #16         @ b1_b2 = { b1, b2 }
291
292   vdup.u16 yyyy1, x1_y1[1]           @ yyyy1 = { yy1, yy1 }
293   vsub.s16 d2_ab, x2_ab, x1_ab
294
295   vdup.u16 yyyy2, x2_y2[1]           @ yyyy2 = { yy2, yy2 }
296   vsub.s16 d1_ab, y2_ab, y1_ab
297
298   orr b0_b1, b0, b1, lsl #16         @ b1_b2 = { b1, b2 }
299   ssub16 dx, x1_x2, x0_x1            @ dx = { x1 - x0, x2 - x1 }
300
301   ssub16 dy, y1_y2, y0_y1            @ dy = { y1 - y0, y2 - y1 }
302   ssub16 db, b1_b2, b0_b1            @ db = { b1 - b0, b2 - b1 }
303
304   vsub.s16 d3_ab, y1_ab, y0_ab
305   smusdx ga_by, dx, db               @ ga_by = ((x1 - x0) * (b2 - b1)) -
306                                      @         ((x2 - X1) * (b1 - b0)) 
307   vmull.s16 ga_uvrg_x, d0_a, d1_a
308   smusdx ga_bx, db, dy               @ ga_bx = ((b1 - b0) * (y2 - y1)) -
309                                      @         ((b2 - b1) * (y1 - y0))
310   vmlsl.s16 ga_uvrg_x, d2_a, d3_a
311   movs gs_bx, ga_bx, asr #31
312
313   vmull.s16 ga_uvrg_y, d0_b, d1_b
314   rsbmi ga_bx, ga_bx, #0
315
316   vmlsl.s16 ga_uvrg_y, d2_b, d3_b
317   movs gs_by, ga_by, asr #31
318
319   vshr.u64 d0, d30, #22
320   mov b_base, b0, lsl #16
321
322   rsbmi ga_by, ga_by, #0
323   vclt.s32 gs_uvrg_x, ga_uvrg_x, #0  @ gs_uvrg_x = ga_uvrg_x < 0
324
325   @ r12 = psx_gpu->triangle_winding_offset
326   ldrb r12, [ psx_gpu, #psx_gpu_triangle_winding_offset ]
327   vclt.s32 gs_uvrg_y, ga_uvrg_y, #0  @ gs_uvrg_y = ga_uvrg_y < 0
328
329   add b_base, b_base, #0x8000
330   rsb r12, r12, #0                   @ r12 = -(triangle->winding)
331
332   vdup.u32 w_mask, r12               @ w_mask = { -w, -w, -w, -w }
333   sub r14, r14, #(62 - 12)           @ r14 = shift - (62 - FIXED_BITS)
334
335   vshll.u16 uvrg_base, uvrg0, #16    @ uvrg_base = uvrg0 << 16
336   vdup.u32 r_shift, r14              @ r_shift = { shift, shift, shift, shift }
337
338   vorr.u32 uvrg_base, #0x8000
339   vabs.s32 ga_uvrg_x, ga_uvrg_x      @ ga_uvrg_x = abs(ga_uvrg_x)
340
341   vmov area_r_s, s0                  @ area_r_s = triangle_reciprocal
342   vabs.s32 ga_uvrg_y, ga_uvrg_y      @ ga_uvrg_y = abs(ga_uvrg_y)
343
344   vmull.u32 gw_rg_x, ga_rg_x, d0[0]
345   vmull.u32 gw_uv_x, ga_uv_x, d0[0]
346   vmull.u32 gw_rg_y, ga_rg_y, d0[0]
347   vmull.u32 gw_uv_y, ga_uv_y, d0[0]
348
349   vshl.u64 gw_rg_x, gw_rg_x, r_shift
350   vshl.u64 gw_uv_x, gw_uv_x, r_shift
351   vshl.u64 gw_rg_y, gw_rg_y, r_shift
352   vshl.u64 gw_uv_y, gw_uv_y, r_shift
353
354   veor.u32 gs_uvrg_x, gs_uvrg_x, w_mask
355   vmovn.u64 g_uv_x, gw_uv_x
356
357   veor.u32 gs_uvrg_y, gs_uvrg_y, w_mask
358   vmovn.u64 g_rg_x, gw_rg_x
359
360   veor.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
361   vmovn.u64 g_uv_y, gw_uv_y
362
363   vsub.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
364   vmovn.u64 g_rg_y, gw_rg_y
365
366   veor.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
367   mov ga_bx, ga_bx, lsl #13
368
369   vsub.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
370   mov ga_by, ga_by, lsl #13
371
372   vdup.u32 x0_y0, x0
373   umull gw_bx_l, gw_bx_h, ga_bx, area_r_s
374
375   vshl.u32 g_uvrg_x, g_uvrg_x, #4
376   vshl.u32 g_uvrg_y, g_uvrg_y, #4
377
378   umull gw_by_l, gw_by_h, ga_by, area_r_s
379   vmls.s32 uvrg_base, ga_uvrg_x, x0_y0[0]
380
381   eor gs_bx, gs_bx, r12
382   vadd.u32 uvrg_dx2, uvrg_dx1, uvrg_dx1
383
384   veor.u32 uvrg_dx0, uvrg_dx0, uvrg_dx0
385   eor gs_by, gs_by, r12
386
387   rsb r11, r14, #0                   @ r11 = negative shift for scalar lsr
388   add store_a, psx_gpu, #psx_gpu_uvrg_offset
389
390   sub r11, r11, #(32 - 13)
391
392   add store_b, store_a, #16
393   mov store_inc, #32
394
395   vadd.u32 uvrg_dx3, uvrg_dx2, uvrg_dx1
396   vst1.u32 { uvrg_base }, [ store_a, : 128 ], store_inc
397
398   vst1.u32 { uvrg_dx1 }, [ store_b, : 128 ], store_inc
399   mov g_bx, gw_bx_h, lsr r11
400
401   vst1.u32 { g_uvrg_y }, [ store_a, : 128 ], store_inc
402   mov g_by, gw_by_h, lsr r11
403
404   vst4.u32 { uvrg_dx0l, uvrg_dx1l, uvrg_dx2l, uvrg_dx3l },                     \
405    [ store_b, : 128 ], store_inc
406   eor g_bx, g_bx, gs_bx
407
408   vst4.u32 { uvrg_dx0h, uvrg_dx1h, uvrg_dx2h, uvrg_dx3h },                     \
409    [ store_b, : 128 ], store_inc
410   sub g_bx, g_bx, gs_bx
411
412   lsl g_bx, g_bx, #4  
413   eor g_by, g_by, gs_by
414
415   mls b_base, g_bx, x0, b_base
416   sub g_by, g_by, gs_by
417
418   lsl g_by, g_by, #4
419   mov g_bx0, #0
420
421   add g_bx2, g_bx, g_bx
422   add g_bx3, g_bx, g_bx2
423
424   stmia store_b, { g_bx0, g_bx, g_bx2, g_bx3, b_base, g_by }
425
426   ldmia sp!, { r4 - r11, pc }
427
428
429 #define psx_gpu                                  r0
430 #define v_a                                      r1
431 #define v_b                                      r2
432 #define v_c                                      r3
433
434 #define temp                                     r14
435
436 #define x_a                                      r4
437 #define x_b                                      r5
438 #define x_c                                      r6
439 #define y_a                                      r1
440 #define y_b                                      r2
441 #define y_c                                      r3
442
443 #define height_minor_a                           r7
444 #define height_minor_b                           r8
445 #define height_major                             r9
446 #define height                                   r9
447
448 #define reciprocal_table_ptr                     r10
449
450 #define edge_alt_low                             r4
451 #define edge_alt_high                            r5
452 #define edge_dx_dy_alt                           r6
453 #define edge_shift_alt                           r10
454
455 #define edge_dx_dy_alt_low                       r4
456 #define edge_dx_dy_alt_high                      r5
457
458 #define span_edge_data                           r4
459 #define span_uvrg_offset                         r5
460 #define span_b_offset                            r6
461
462 #define clip                                     r14
463
464 #define b                                        r11
465 #define b_dy                                     r12
466
467
468 #define alternate_x                              q0
469 #define alternate_dx_dy                          q1
470 #define alternate_x_32                           q2
471
472 #define alternate_x_low                          d0
473 #define alternate_x_high                         d1
474 #define alternate_dx_dy_low                      d2
475 #define alternate_dx_dy_high                     d3
476 #define alternate_x_32_low                       d4
477 #define alternate_x_32_high                      d5
478
479 #define left_x                                   q3
480 #define right_x                                  q4
481 #define left_dx_dy                               q5
482 #define right_dx_dy                              q6
483 #define left_edge                                q7
484 #define right_edge                               q8
485
486 #define left_x_low                               d6
487 #define left_x_high                              d7
488 #define right_x_low                              d8
489 #define right_x_high                             d9
490 #define left_dx_dy_low                           d10
491 #define left_dx_dy_high                          d11
492 #define right_dx_dy_low                          d12
493 #define right_dx_dy_high                         d13
494 #define left_edge_low                            d14
495 #define left_edge_high                           d15
496 #define right_edge_low                           d16
497 #define right_edge_high                          d17
498
499 #define y_mid_point                              d18
500 #define c_0x0004                                 d19
501
502 #define left_right_x_16                          q11
503 #define span_shifts_y                            q12
504 #define c_0x0001                                 q13
505
506 #define span_shifts                              d24
507 #define y_x4                                     d25
508 #define c_0xFFFE                                 d26
509 #define c_0x0007                                 d27
510
511 #define left_right_x_16_low                      d22
512 #define left_right_x_16_high                     d23
513
514 #define uvrg                                     q14
515 #define uvrg_dy                                  q15
516
517 #define alternate_x_16                           d4
518
519 #define v_clip                                   q3
520 #define v_clip_low                               d6
521
522 #define right_x_32                               q10
523 #define left_x_32                                q11
524 #define alternate_select                         d24
525
526 #define right_x_32_low                           d20
527 #define right_x_32_high                          d21
528 #define left_x_32_low                            d22
529 #define left_x_32_high                           d23
530
531 #define edges_xy                                 q0
532 #define edges_dx_dy                              d2
533 #define edge_shifts                              d3
534 #define edge_shifts_64                           q2
535
536 #define edges_xy_left                            d0
537 #define edges_xy_right                           d1
538
539 #define height_reciprocals                       d6
540 #define heights                                  d7
541
542 #define widths                                   d8
543 #define c_0x01                                   d9
544 #define x_starts                                 d10
545 #define x_ends                                   d11
546
547 #define heights_b                                d12
548 #define edges_dx_dy_64                           q10
549
550 #define edges_dx_dy_64_left                      d20
551 #define edges_dx_dy_64_right                     d21
552
553
554 #define setup_spans_prologue()                                                 \
555   stmdb sp!, { r4 - r11, lr };                                                 \
556                                                                                \
557   ldrsh x_a, [ v_a, #8 ];                                                      \
558   ldrsh x_b, [ v_b, #8 ];                                                      \
559   ldrsh x_c, [ v_c, #8 ];                                                      \
560   ldrsh y_a, [ v_a, #10 ];                                                     \
561   ldrsh y_b, [ v_b, #10 ];                                                     \
562   ldrsh y_c, [ v_c, #10 ];                                                     \
563                                                                                \
564   add temp, psx_gpu, #psx_gpu_uvrg_offset;                                     \
565   vld1.32 { uvrg }, [ temp ];                                                  \
566   add temp, psx_gpu, #psx_gpu_uvrg_dy_offset;                                  \
567   vld1.32 { uvrg_dy }, [ temp ];                                               \
568   load_pointer(reciprocal_table_ptr, reciprocal_table);                        \
569                                                                                \
570   vmov.u32 c_0x01, #0x01                                                       \
571
572 #define setup_spans_load_b()                                                   \
573   ldr b, [ psx_gpu, #psx_gpu_b_offset ];                                       \
574   ldr b_dy, [ psx_gpu, #psx_gpu_b_dy_offset ]                                  \
575
576 #define setup_spans_prologue_b()                                               \
577   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
578   add temp, psx_gpu, #psx_gpu_viewport_start_x_offset;                         \
579                                                                                \
580   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
581   vmov.u16 c_0x0004, #0x0004;                                                  \
582                                                                                \
583   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
584   vmov.u16 c_0x0001, #0x0001;                                                  \
585                                                                                \
586   vld1.u16 { left_edge_low[], left_edge_high[] }, [ temp ];                    \
587   add temp, psx_gpu, #psx_gpu_viewport_end_x_offset;                           \
588                                                                                \
589   vld1.u16 { right_edge_low[], right_edge_high[] }, [ temp ];                  \
590   vadd.u16 right_edge, right_edge, c_0x0001;                                   \
591                                                                                \
592   vmov.u16 c_0x0007, #0x0007;                                                  \
593   vmvn.u16 c_0xFFFE, #0x0001                                                   \
594
595
596 #define compute_edge_delta_x2()                                                \
597   ldr temp, [ reciprocal_table_ptr, height, lsl #2 ];                          \
598                                                                                \
599   vdup.u32 heights, height;                                                    \
600   vsub.u32 widths, x_ends, x_starts;                                           \
601                                                                                \
602   vdup.u32 edge_shifts, temp;                                                  \
603   vsub.u32 heights_b, heights, c_0x01;                                         \
604   vshr.u32 height_reciprocals, edge_shifts, #10;                               \
605                                                                                \
606   vmla.s32 heights_b, x_starts, heights;                                       \
607   vbic.u16 edge_shifts, #0xE0;                                                 \
608   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
609   vmull.s32 edges_xy, heights_b, height_reciprocals                            \
610
611 #define width_alt                 r6
612 #define height_reciprocal_alt     r11
613 #define height_b_alt              r12
614
615 #define compute_edge_delta_x3(start_c, height_a, height_b)                     \
616   vmov.u32 heights, height_a, height_b;                                        \
617   ldr temp, [ reciprocal_table_ptr, height_a, lsl #2 ];                        \
618   vmov.u32 edge_shifts[0], temp;                                               \
619   ldr temp, [ reciprocal_table_ptr, height_b, lsl #2 ];                        \
620   vmov.u32 edge_shifts[1], temp;                                               \
621   ldr edge_shift_alt, [ reciprocal_table_ptr, height_minor_b, lsl #2 ];        \
622                                                                                \
623   vsub.u32 widths, x_ends, x_starts;                                           \
624   sub width_alt, x_c, start_c;                                                 \
625                                                                                \
626   vsub.u32 heights_b, heights, c_0x01;                                         \
627   sub height_b_alt, height_minor_b, #1;                                        \
628                                                                                \
629   vshr.u32 height_reciprocals, edge_shifts, #10;                               \
630   lsr height_reciprocal_alt, edge_shift_alt, #10;                              \
631                                                                                \
632   vmla.s32 heights_b, x_starts, heights;                                       \
633   mla height_b_alt, height_minor_b, start_c, height_b_alt;                     \
634                                                                                \
635   vbic.u16 edge_shifts, #0xE0;                                                 \
636   and edge_shift_alt, edge_shift_alt, #0x1F;                                   \
637                                                                                \
638   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
639   mul edge_dx_dy_alt, width_alt, height_reciprocal_alt;                        \
640                                                                                \
641   vmull.s32 edges_xy, heights_b, height_reciprocals;                           \
642   smull edge_alt_low, edge_alt_high, height_b_alt, height_reciprocal_alt       \
643
644
645 #define setup_spans_adjust_y_up()                                              \
646   vsub.u32 y_x4, y_x4, c_0x0004                                                \
647
648 #define setup_spans_adjust_y_down()                                            \
649   vadd.u32 y_x4, y_x4, c_0x0004                                                \
650
651 #define setup_spans_adjust_interpolants_up()                                   \
652   vsub.u32 uvrg, uvrg, uvrg_dy;                                                \
653   sub b, b, b_dy                                                               \
654
655 #define setup_spans_adjust_interpolants_down()                                 \
656   vadd.u32 uvrg, uvrg, uvrg_dy;                                                \
657   add b, b, b_dy                                                               \
658
659
660 #define setup_spans_clip_interpolants_increment()                              \
661   mla b, b_dy, clip, b;                                                        \
662   vmla.s32 uvrg, uvrg_dy, v_clip                                               \
663
664 #define setup_spans_clip_interpolants_decrement()                              \
665   mls b, b_dy, clip, b;                                                        \
666   vmls.s32 uvrg, uvrg_dy, v_clip                                               \
667
668 #define setup_spans_clip_alternate_yes()                                       \
669   smlal edge_alt_low, edge_alt_high, edge_dx_dy_alt, clip                      \
670
671 #define setup_spans_clip_alternate_no()                                        \
672
673 #define setup_spans_clip(direction, alternate_active)                          \
674   vdup.u32 v_clip, clip;                                                       \
675   setup_spans_clip_alternate_##alternate_active();                             \
676   setup_spans_clip_interpolants_##direction();                                 \
677   vmlal.s32 edges_xy, edges_dx_dy, v_clip_low                                  \
678
679
680 #define setup_spans_adjust_edges_alternate_no(left_index, right_index)         \
681   vmovl.s32 edge_shifts_64, edge_shifts;                                       \
682   vmovl.s32 edges_dx_dy_64, edges_dx_dy;                                       \
683                                                                                \
684   vshl.s64 edges_xy, edges_xy, edge_shifts_64;                                 \
685   vshl.s64 edges_dx_dy_64, edges_dx_dy_64, edge_shifts_64;                     \
686                                                                                \
687   vmov left_x_low, edges_xy_##left_index;                                      \
688   vmov right_x_low, edges_xy_##right_index;                                    \
689                                                                                \
690   vmov left_dx_dy_low, edges_dx_dy_64_##left_index;                            \
691   vmov left_dx_dy_high, edges_dx_dy_64_##left_index;                           \
692   vmov right_dx_dy_low, edges_dx_dy_64_##right_index;                          \
693   vmov right_dx_dy_high, edges_dx_dy_64_##right_index;                         \
694                                                                                \
695   vadd.u64 left_x_high, left_x_low, left_dx_dy_low;                            \
696   vadd.u64 right_x_high, right_x_low, right_dx_dy_low;                         \
697                                                                                \
698   vadd.u64 left_dx_dy, left_dx_dy, left_dx_dy;                                 \
699   vadd.u64 right_dx_dy, right_dx_dy, right_dx_dy                               \
700
701
702 #define setup_spans_adjust_edges_alternate_yes(left_index, right_index)        \
703   setup_spans_adjust_edges_alternate_no(left_index, right_index);              \
704                                                                                \
705   vdup.u16 y_mid_point, y_b;                                                   \
706   rsb temp, edge_shift_alt, #32;                                               \
707                                                                                \
708   lsl edge_alt_high, edge_alt_high, edge_shift_alt;                            \
709   orr edge_alt_high, edge_alt_high, edge_alt_low, lsr temp;                    \
710   lsl edge_alt_low, edge_alt_low, edge_shift_alt;                              \
711   vmov alternate_x_low, edge_alt_low, edge_alt_high;                           \
712                                                                                \
713   asr edge_dx_dy_alt_high, edge_dx_dy_alt, temp;                               \
714   lsl edge_dx_dy_alt_low, edge_dx_dy_alt, edge_shift_alt;                      \
715   vmov alternate_dx_dy_low, edge_dx_dy_alt_low, edge_dx_dy_alt_high;           \
716   vmov alternate_dx_dy_high, alternate_dx_dy_low;                              \
717                                                                                \
718   vadd.u64 alternate_x_high, alternate_x_low, alternate_dx_dy_low;             \
719   vadd.u64 alternate_dx_dy, alternate_dx_dy, alternate_dx_dy                   \
720
721
722 #define setup_spans_y_select_up()                                              \
723   vclt.s16 alternate_select, y_x4, y_mid_point                                 \
724
725 #define setup_spans_y_select_down()                                            \
726   vcgt.s16 alternate_select, y_x4, y_mid_point                                 \
727
728
729 #define setup_spans_alternate_select_left()                                    \
730   vbit.u16 left_right_x_16_low, alternate_x_16, alternate_select               \
731
732 #define setup_spans_alternate_select_right()                                   \
733   vbit.u16 left_right_x_16_high, alternate_x_16, alternate_select              \
734
735
736 #define setup_spans_set_x4_alternate_yes(alternate, direction)                 \
737   vshrn.s64 alternate_x_32_low, alternate_x, #32;                              \
738   vshrn.s64 left_x_32_low, left_x, #32;                                        \
739   vshrn.s64 right_x_32_low, right_x, #32;                                      \
740                                                                                \
741   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
742   vadd.u64 left_x, left_x, left_dx_dy;                                         \
743   vadd.u64 right_x, right_x, right_dx_dy;                                      \
744                                                                                \
745   vshrn.s64 alternate_x_32_high, alternate_x, #32;                             \
746   vshrn.s64 left_x_32_high, left_x, #32;                                       \
747   vshrn.s64 right_x_32_high, right_x, #32;                                     \
748                                                                                \
749   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
750   vadd.u64 left_x, left_x, left_dx_dy;                                         \
751   vadd.u64 right_x, right_x, right_dx_dy;                                      \
752                                                                                \
753   vmovn.u32 alternate_x_16, alternate_x_32;                                    \
754   setup_spans_y_select_##direction();                                          \
755   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
756                                                                                \
757   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
758   setup_spans_alternate_select_##alternate();                                  \
759                                                                                \
760   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
761   str b, [ span_b_offset ], #4;                                                \
762   setup_spans_adjust_interpolants_##direction();                               \
763                                                                                \
764   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
765                                                                                \
766   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
767   str b, [ span_b_offset ], #4;                                                \
768   setup_spans_adjust_interpolants_##direction();                               \
769                                                                                \
770   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
771                                                                                \
772   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
773   str b, [ span_b_offset ], #4;                                                \
774   setup_spans_adjust_interpolants_##direction();                               \
775                                                                                \
776   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
777   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
778   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
779                                                                                \
780   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
781   str b, [ span_b_offset ], #4;                                                \
782   setup_spans_adjust_interpolants_##direction();                               \
783                                                                                \
784   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
785   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
786                                                                                \
787   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
788                                                                                \
789   setup_spans_adjust_y_##direction()                                           \
790
791
792 #define setup_spans_set_x4_alternate_no(alternate, direction)                  \
793   vshrn.s64 left_x_32_low, left_x, #32;                                        \
794   vshrn.s64 right_x_32_low, right_x, #32;                                      \
795                                                                                \
796   vadd.u64 left_x, left_x, left_dx_dy;                                         \
797   vadd.u64 right_x, right_x, right_dx_dy;                                      \
798                                                                                \
799   vshrn.s64 left_x_32_high, left_x, #32;                                       \
800   vshrn.s64 right_x_32_high, right_x, #32;                                     \
801                                                                                \
802   vadd.u64 left_x, left_x, left_dx_dy;                                         \
803   vadd.u64 right_x, right_x, right_dx_dy;                                      \
804                                                                                \
805   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
806   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
807                                                                                \
808   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
809   str b, [ span_b_offset ], #4;                                                \
810   setup_spans_adjust_interpolants_##direction();                               \
811                                                                                \
812   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
813                                                                                \
814   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
815   str b, [ span_b_offset ], #4;                                                \
816   setup_spans_adjust_interpolants_##direction();                               \
817                                                                                \
818   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
819                                                                                \
820   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
821   str b, [ span_b_offset ], #4;                                                \
822   setup_spans_adjust_interpolants_##direction();                               \
823                                                                                \
824   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
825   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
826   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
827                                                                                \
828   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
829   str b, [ span_b_offset ], #4;                                                \
830   setup_spans_adjust_interpolants_##direction();                               \
831                                                                                \
832   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
833   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
834                                                                                \
835   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
836                                                                                \
837   setup_spans_adjust_y_##direction()                                           \
838
839
840 #define edge_adjust_low           r11
841 #define edge_adjust_high          r12
842
843 #define setup_spans_alternate_adjust_yes()                                     \
844   smull edge_adjust_low, edge_adjust_high, edge_dx_dy_alt, height_minor_a;     \
845   subs edge_alt_low, edge_alt_low, edge_adjust_low;                            \
846   sbc edge_alt_high, edge_alt_high, edge_adjust_high                           \
847
848 #define setup_spans_alternate_adjust_no()                                      \
849
850
851 #define setup_spans_down(left_index, right_index, alternate, alternate_active) \
852   setup_spans_alternate_adjust_##alternate_active();                           \
853   setup_spans_load_b();                                                        \
854                                                                                \
855   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                     \
856   subs y_c, y_c, temp;                                                         \
857   subgt height, height, y_c;                                                   \
858   addgt height, height, #1;                                                    \
859                                                                                \
860   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                   \
861   subs clip, temp, y_a;                                                        \
862   ble 0f;                                                                      \
863                                                                                \
864   sub height, height, clip;                                                    \
865   add y_a, y_a, clip;                                                          \
866   setup_spans_clip(increment, alternate_active);                               \
867                                                                                \
868  0:                                                                            \
869   cmp height, #0;                                                              \
870   ble 1f;                                                                      \
871                                                                                \
872   orr temp, y_a, y_a, lsl #16;                                                 \
873   add temp, temp, #(1 << 16);                                                  \
874   add y_a, temp, #2;                                                           \
875   add y_a, y_a, #(2 << 16);                                                    \
876   vmov.u32 y_x4, temp, y_a;                                                    \
877                                                                                \
878   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
879    right_index);                                                               \
880   setup_spans_prologue_b();                                                    \
881                                                                                \
882   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
883                                                                                \
884  2:                                                                            \
885   setup_spans_set_x4_alternate_##alternate_active(alternate, down);            \
886   subs height, height, #4;                                                     \
887   bhi 2b;                                                                      \
888                                                                                \
889  1:                                                                            \
890
891
892 #define setup_spans_alternate_pre_increment_yes()                              \
893   adds edge_alt_low, edge_alt_low, edge_dx_dy_alt;                             \
894   adc edge_alt_high, edge_alt_high, edge_dx_dy_alt, asr #31                    \
895
896 #define setup_spans_alternate_pre_increment_no()                               \
897
898
899 #define setup_spans_up_decrement_yes()                                         \
900   suble height, height, #1                                                     \
901
902 #define setup_spans_up_decrement_no()                                          \
903
904
905 #define setup_spans_up(left_index, right_index, alternate, alternate_active)   \
906   setup_spans_alternate_adjust_##alternate_active();                           \
907   setup_spans_load_b();                                                        \
908   sub y_a, y_a, #1;                                                            \
909                                                                                \
910   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                    \
911   subs temp, temp, y_c;                                                        \
912   subgt height, height, temp;                                                  \
913   setup_spans_up_decrement_##alternate_active();                               \
914                                                                                \
915   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                      \
916   subs clip, y_a, temp;                                                        \
917   ble 0f;                                                                      \
918                                                                                \
919   sub height, height, clip;                                                    \
920   sub y_a, y_a, clip;                                                          \
921   setup_spans_clip(decrement, alternate_active);                               \
922                                                                                \
923  0:                                                                            \
924   cmp height, #0;                                                              \
925   ble 1f;                                                                      \
926                                                                                \
927   orr temp, y_a, y_a, lsl #16;                                                 \
928   sub temp, temp, #(1 << 16);                                                  \
929   sub y_a, temp, #2;                                                           \
930   sub y_a, y_a, #(2 << 16);                                                    \
931   vmov.u32 y_x4, temp, y_a;                                                    \
932                                                                                \
933   vaddw.s32 edges_xy, edges_xy, edges_dx_dy;                                   \
934                                                                                \
935   setup_spans_alternate_pre_increment_##alternate_active();                    \
936   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
937    right_index);                                                               \
938   setup_spans_adjust_interpolants_up();                                        \
939   setup_spans_prologue_b();                                                    \
940                                                                                \
941   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
942                                                                                \
943  2:                                                                            \
944   setup_spans_set_x4_alternate_##alternate_active(alternate, up);              \
945   subs height, height, #4;                                                     \
946   bhi 2b;                                                                      \
947                                                                                \
948  1:                                                                            \
949
950
951 #define setup_spans_epilogue()                                                 \
952   ldmia sp!, { r4 - r11, pc }                                                  \
953
954
955 #define setup_spans_up_up(minor, major)                                        \
956   setup_spans_prologue();                                                      \
957   sub height_minor_a, y_a, y_b;                                                \
958   sub height_minor_b, y_b, y_c;                                                \
959   sub height, y_a, y_c;                                                        \
960                                                                                \
961   vdup.u32 x_starts, x_a;                                                      \
962   vmov.u32 x_ends, x_c, x_b;                                                   \
963                                                                                \
964   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
965   setup_spans_up(major, minor, minor, yes);                                    \
966   setup_spans_epilogue()                                                       \
967
968 function(setup_spans_up_left)
969   setup_spans_up_up(left, right)
970
971 function(setup_spans_up_right)
972   setup_spans_up_up(right, left)
973
974 .pool
975
976 #define setup_spans_down_down(minor, major)                                    \
977   setup_spans_prologue();                                                      \
978   sub height_minor_a, y_b, y_a;                                                \
979   sub height_minor_b, y_c, y_b;                                                \
980   sub height, y_c, y_a;                                                        \
981                                                                                \
982   vdup.u32 x_starts, x_a;                                                      \
983   vmov.u32 x_ends, x_c, x_b;                                                   \
984                                                                                \
985   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
986   setup_spans_down(major, minor, minor, yes);                                  \
987   setup_spans_epilogue()                                                       \
988
989 function(setup_spans_down_left)
990   setup_spans_down_down(left, right)
991
992 function(setup_spans_down_right)
993   setup_spans_down_down(right, left)
994
995
996 #define setup_spans_up_flat()                                                  \
997   sub height, y_a, y_c;                                                        \
998                                                                                \
999   compute_edge_delta_x2();                                                     \
1000   setup_spans_up(left, right, none, no);                                       \
1001   setup_spans_epilogue()                                                       \
1002
1003 function(setup_spans_up_a)
1004   setup_spans_prologue()
1005
1006   vmov.u32 x_starts, x_a, x_b
1007   vdup.u32 x_ends, x_c
1008
1009   setup_spans_up_flat()
1010
1011 function(setup_spans_up_b)
1012   setup_spans_prologue()
1013
1014   vdup.u32 x_starts, x_a
1015   vmov.u32 x_ends, x_b, x_c
1016
1017   setup_spans_up_flat()
1018
1019 #define setup_spans_down_flat()                                                \
1020   sub height, y_c, y_a;                                                        \
1021                                                                                \
1022   compute_edge_delta_x2();                                                     \
1023   setup_spans_down(left, right, none, no);                                     \
1024   setup_spans_epilogue()                                                       \
1025
1026 function(setup_spans_down_a)
1027   setup_spans_prologue()
1028
1029   vmov.u32 x_starts, x_a, x_b
1030   vdup.u32 x_ends, x_c
1031
1032   setup_spans_down_flat()
1033
1034 function(setup_spans_down_b)
1035   setup_spans_prologue()
1036
1037   vdup.u32 x_starts, x_a
1038   vmov.u32 x_ends, x_b, x_c
1039
1040   setup_spans_down_flat()
1041
1042
1043 #define middle_y                                          r9
1044
1045 #define edges_xy_b                                        q11
1046 #define edges_dx_dy_b                                     d26
1047 #define edge_shifts_b                                     d27
1048 #define edges_dx_dy_and_shifts_b                          q13
1049 #define height_increment                                  d20
1050
1051 #define edges_dx_dy_and_shifts                            q1
1052
1053 #define edges_xy_b_left                                   d22
1054 #define edges_xy_b_right                                  d23
1055
1056 #define setup_spans_up_down_load_edge_set_b()                                  \
1057   vmov edges_xy, edges_xy_b;                                                   \
1058   vmov edges_dx_dy_and_shifts, edges_dx_dy_and_shifts_b                        \
1059
1060
1061 function(setup_spans_up_down)
1062   setup_spans_prologue()
1063
1064   // s32 middle_y = y_a;
1065   sub height_minor_a, y_a, y_b
1066   sub height_minor_b, y_c, y_a
1067   sub height_major, y_c, y_b
1068
1069   vmov.u32 x_starts, x_a, x_c
1070   vdup.u32 x_ends, x_b
1071
1072   compute_edge_delta_x3(x_a, height_minor_a, height_major)
1073
1074   mov temp, #0
1075   vmov.u32 height_increment, temp, height_minor_b
1076   vmlal.s32 edges_xy, edges_dx_dy, height_increment
1077
1078   vmov edges_xy_b_left, edge_alt_low, edge_alt_high
1079   vmov edges_xy_b_right, edges_xy_right
1080
1081   vmov edge_shifts_b, edge_shifts
1082   vmov.u32 edge_shifts_b[0], edge_shift_alt
1083
1084   vneg.s32 edges_dx_dy_b, edges_dx_dy
1085   vmov.u32 edges_dx_dy_b[0], edge_dx_dy_alt
1086
1087   mov middle_y, y_a
1088   
1089   setup_spans_load_b()
1090   sub y_a, y_a, #1
1091
1092   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1093   subs temp, temp, y_b
1094   subgt height_minor_a, height_minor_a, temp
1095
1096   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1097   subs clip, y_a, temp
1098   ble 0f
1099
1100   sub height_minor_a, height_minor_a, clip
1101   sub y_a, y_a, clip
1102   setup_spans_clip(decrement, no)
1103
1104  0:                                                                
1105   cmp height_minor_a, #0
1106   ble 3f
1107
1108   orr temp, y_a, y_a, lsl #16
1109   sub temp, temp, #(1 << 16)
1110   sub y_a, temp, #2
1111   sub y_a, y_a, #(2 << 16)
1112   vmov.u32 y_x4, temp, y_a
1113
1114   vaddw.s32 edges_xy, edges_xy, edges_dx_dy
1115
1116   strh height_minor_a, [ psx_gpu, #psx_gpu_num_spans_offset ]
1117
1118   setup_spans_adjust_edges_alternate_no(left, right); 
1119   setup_spans_adjust_interpolants_up()
1120   setup_spans_up_down_load_edge_set_b()
1121
1122   setup_spans_prologue_b()
1123
1124
1125  2: 
1126   setup_spans_set_x4_alternate_no(none, up)
1127   subs height_minor_a, height_minor_a, #4
1128   bhi 2b
1129
1130   add span_edge_data, span_edge_data, height_minor_a, lsl #3
1131   add span_uvrg_offset, span_uvrg_offset, height_minor_a, lsl #4
1132   add span_b_offset, span_b_offset, height_minor_a, lsl #2
1133
1134  4:
1135   add temp, psx_gpu, #psx_gpu_uvrg_offset
1136   vld1.32 { uvrg }, [ temp ]
1137   mov y_a, middle_y
1138   
1139   setup_spans_load_b()
1140
1141   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1142   subs y_c, y_c, temp
1143   subgt height_minor_b, height_minor_b, y_c
1144   addgt height_minor_b, height_minor_b, #1
1145
1146   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1147   subs clip, temp, y_a
1148   ble 0f
1149
1150   sub height_minor_b, height_minor_b, clip
1151   add y_a, y_a, clip
1152   setup_spans_clip(increment, no)
1153
1154  0:
1155   cmp height_minor_b, #0
1156   ble 1f
1157
1158   orr temp, y_a, y_a, lsl #16
1159   add temp, temp, #(1 << 16) 
1160   add y_a, temp, #2
1161   add y_a, y_a, #(2 << 16)
1162   vmov.u32 y_x4, temp, y_a
1163
1164   setup_spans_adjust_edges_alternate_no(left, right)
1165
1166   ldrh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1167   add temp, temp, height_minor_b
1168   strh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1169
1170  2:                                                     
1171   setup_spans_set_x4_alternate_no(none, down)
1172   subs height_minor_b, height_minor_b, #4
1173   bhi 2b
1174
1175  1:
1176   setup_spans_epilogue()
1177
1178  3:
1179   setup_spans_up_down_load_edge_set_b()
1180   setup_spans_prologue_b()
1181   bal 4b
1182
1183 .pool
1184
1185 #undef span_uvrg_offset
1186 #undef span_edge_data
1187 #undef span_b_offset
1188 #undef left_x
1189 #undef b
1190
1191 #define psx_gpu                                  r0
1192 #define num_spans                                r1
1193 #define span_uvrg_offset                         r2
1194 #define span_edge_data                           r3
1195 #define span_b_offset                            r4
1196 #define b_dx                                     r5
1197 #define span_num_blocks                          r6
1198 #define y                                        r7
1199 #define left_x                                   r8
1200 #define b                                        r9
1201 #define dither_offset_ptr                        r10
1202 #define block_ptr_a                              r11
1203 #define fb_ptr                                   r12
1204 #define num_blocks                               r14
1205
1206 #define uvrg_dx_ptr                              r2
1207 #define texture_mask_ptr                         r3
1208 #define dither_shift                             r8
1209 #define dither_row                               r10
1210
1211 #define c_32                                     r7
1212 #define b_dx4                                    r8
1213 #define b_dx8                                    r9
1214 #define block_ptr_b                              r10
1215
1216 #define block_span_ptr                           r10
1217 #define right_mask                               r8
1218
1219 #define color                                    r2
1220 #define color_r                                  r3
1221 #define color_g                                  r4
1222 #define color_b                                  r5
1223
1224 #undef uvrg
1225
1226 #define u_block                                  q0
1227 #define v_block                                  q1
1228 #define r_block                                  q2
1229 #define g_block                                  q3
1230 #define b_block                                  q4
1231
1232 #define uv_dx4                                   d10
1233 #define rg_dx4                                   d11
1234 #define uv_dx8                                   d12
1235 #define rg_dx8                                   d13
1236 #define b_whole_8                                d14
1237 #define fb_mask_ptrs                             d15
1238
1239 #define uvrg_dx4                                 q5
1240 #define uvrg_dx8                                 q6
1241 #define uv_dx8                                   d12
1242 #define rg_dx8                                   d13
1243
1244 #define u_whole                                  q8
1245 #define v_whole                                  q9
1246 #define r_whole                                  q10
1247 #define g_whole                                  q11
1248 #define b_whole                                  q12
1249
1250 #define u_whole_low                              d16
1251 #define u_whole_high                             d17
1252 #define v_whole_low                              d18
1253 #define v_whole_high                             d19
1254 #define r_whole_low                              d20
1255 #define r_whole_high                             d21
1256 #define g_whole_low                              d22
1257 #define g_whole_high                             d23
1258 #define b_whole_low                              d24
1259 #define b_whole_high                             d25
1260
1261 #define dx4                                      q13
1262 #define dx8                                      q13
1263
1264 #define u_whole_8                                d26
1265 #define v_whole_8                                d27
1266 #define u_whole_8b                               d24
1267 #define r_whole_8                                d24
1268 #define g_whole_8                                d25
1269
1270 #define uv_whole_8                               q13
1271 #define uv_whole_8b                              q14
1272
1273 #define dither_offsets                           q14
1274 #define texture_mask                             q15
1275 #define texture_mask_u                           d30
1276 #define texture_mask_v                           d31
1277
1278 #define dither_offsets_short                     d28
1279
1280 #define v_left_x                                 q8
1281 #define uvrg                                     q9
1282 #define block_span                               q10
1283
1284 #define uv                                       d18
1285 #define rg                                       d19
1286
1287 #define draw_mask                                q1
1288 #define draw_mask_edge                           q13
1289 #define test_mask                                q0
1290
1291 #define uvrg_dx                                  q3
1292
1293 #define colors                                   q2
1294
1295 #define setup_blocks_texture_swizzled()                                        \
1296   vand.u8 u_whole_8b, u_whole_8, texture_mask_u;                               \
1297   vsli.u8 u_whole_8, v_whole_8, #4;                                            \
1298   vsri.u8 v_whole_8, u_whole_8b, #4                                            \
1299
1300 #define setup_blocks_texture_unswizzled()                                      \
1301
1302
1303 #define setup_blocks_shaded_textured_builder(swizzling)                        \
1304 .align 3;                                                                      \
1305                                                                                \
1306 function(setup_blocks_shaded_textured_dithered_##swizzling##_indirect)         \
1307   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1308   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1309                                                                                \
1310   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1311   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1312                                                                                \
1313   cmp num_spans, #0;                                                           \
1314   bxeq lr;                                                                     \
1315                                                                                \
1316   stmdb sp!, { r4 - r11, r14 };                                                \
1317   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1318                                                                                \
1319   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
1320   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1321                                                                                \
1322   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1323   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1324                                                                                \
1325   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1326   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1327                                                                                \
1328   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
1329   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1330                                                                                \
1331   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1332                                                                                \
1333  0:                                                                            \
1334   vmov.u8 fb_mask_ptrs, #0;                                                    \
1335                                                                                \
1336   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1337   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1338                                                                                \
1339   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1340   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1341                                                                                \
1342   cmp span_num_blocks, #0;                                                     \
1343   beq 1f;                                                                      \
1344                                                                                \
1345   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1346   add num_blocks, span_num_blocks, num_blocks;                                 \
1347                                                                                \
1348   cmp num_blocks, #MAX_BLOCKS;                                                 \
1349   bgt 2f;                                                                      \
1350                                                                                \
1351  3:                                                                            \
1352   ldr b, [ span_b_offset ];                                                    \
1353   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1354                                                                                \
1355   vdup.u32 v_left_x, left_x;                                                   \
1356   and y, y, #0x3;                                                              \
1357                                                                                \
1358   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1359   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1360                                                                                \
1361   mla b, b_dx, left_x, b;                                                      \
1362   and dither_shift, left_x, #0x03;                                             \
1363                                                                                \
1364   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1365   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1366                                                                                \
1367   mov dither_shift, dither_shift, lsl #3;                                      \
1368   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1369                                                                                \
1370   mov c_32, #32;                                                               \
1371   subs span_num_blocks, span_num_blocks, #1;                                   \
1372                                                                                \
1373   mov dither_row, dither_row, ror dither_shift;                                \
1374   mov b_dx4, b_dx, lsl #2;                                                     \
1375                                                                                \
1376   vdup.u32 dither_offsets_short, dither_row;                                   \
1377   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1378                                                                                \
1379   vdup.u32 b_block, b;                                                         \
1380   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1381                                                                                \
1382   vdup.u32 u_block, uv[0];                                                     \
1383   mov b_dx8, b_dx, lsl #3;                                                     \
1384                                                                                \
1385   vdup.u32 v_block, uv[1];                                                     \
1386   vdup.u32 r_block, rg[0];                                                     \
1387   vdup.u32 g_block, rg[1];                                                     \
1388                                                                                \
1389   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1390                                                                                \
1391   vadd.u32 u_block, u_block, block_span;                                       \
1392   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1393                                                                                \
1394   vadd.u32 v_block, v_block, block_span;                                       \
1395   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1396                                                                                \
1397   vadd.u32 r_block, r_block, block_span;                                       \
1398   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1399                                                                                \
1400   vadd.u32 g_block, g_block, block_span;                                       \
1401   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
1402                                                                                \
1403   vadd.u32 b_block, b_block, block_span;                                       \
1404   add block_ptr_b, block_ptr_a, #16;                                           \
1405                                                                                \
1406   vshrn.u32 u_whole_low, u_block, #16;                                         \
1407   vshrn.u32 v_whole_low, v_block, #16;                                         \
1408   vshrn.u32 r_whole_low, r_block, #16;                                         \
1409   vshrn.u32 g_whole_low, g_block, #16;                                         \
1410                                                                                \
1411   vdup.u32 dx4, uv_dx4[0];                                                     \
1412   vshrn.u32 b_whole_low, b_block, #16;                                         \
1413                                                                                \
1414   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1415   vdup.u32 dx4, uv_dx4[1];                                                     \
1416                                                                                \
1417   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1418   vdup.u32 dx4, rg_dx4[0];                                                     \
1419                                                                                \
1420   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1421   vdup.u32 dx4, rg_dx4[1];                                                     \
1422                                                                                \
1423   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1424   vdup.u32 dx4, b_dx4;                                                         \
1425                                                                                \
1426   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1427   vdup.u32 dx8, uv_dx8[0];                                                     \
1428                                                                                \
1429   vadd.u32 u_block, u_block, dx8;                                              \
1430   vdup.u32 dx8, uv_dx8[1];                                                     \
1431                                                                                \
1432   vadd.u32 v_block, v_block, dx8;                                              \
1433   vdup.u32 dx8, rg_dx8[0];                                                     \
1434                                                                                \
1435   vadd.u32 r_block, r_block, dx8;                                              \
1436   vdup.u32 dx8, rg_dx8[1];                                                     \
1437                                                                                \
1438   vadd.u32 g_block, g_block, dx8;                                              \
1439   vdup.u32 dx8, b_dx8;                                                         \
1440                                                                                \
1441   vadd.u32 b_block, b_block, dx8;                                              \
1442   vmovn.u16 u_whole_8, u_whole;                                                \
1443                                                                                \
1444   vmovn.u16 v_whole_8, v_whole;                                                \
1445                                                                                \
1446   vmovn.u16 b_whole_8, b_whole;                                                \
1447   pld [ fb_ptr ];                                                              \
1448   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1449                                                                                \
1450   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1451   setup_blocks_texture_##swizzling();                                          \
1452                                                                                \
1453   vmovn.u16 r_whole_8, r_whole;                                                \
1454   beq 5f;                                                                      \
1455                                                                                \
1456  4:                                                                            \
1457   vmovn.u16 g_whole_8, g_whole;                                                \
1458   vshrn.u32 u_whole_low, u_block, #16;                                         \
1459                                                                                \
1460   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1461   vshrn.u32 v_whole_low, v_block, #16;                                         \
1462                                                                                \
1463   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1464   vshrn.u32 r_whole_low, r_block, #16;                                         \
1465                                                                                \
1466   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1467   vshrn.u32 g_whole_low, g_block, #16;                                         \
1468                                                                                \
1469   vdup.u32 dx4, uv_dx4[0];                                                     \
1470   vshrn.u32 b_whole_low, b_block, #16;                                         \
1471                                                                                \
1472   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1473   vdup.u32 dx4, uv_dx4[1];                                                     \
1474                                                                                \
1475   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1476   vdup.u32 dx4, rg_dx4[0];                                                     \
1477                                                                                \
1478   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1479   vdup.u32 dx4, rg_dx4[1];                                                     \
1480                                                                                \
1481   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1482   vdup.u32 dx4, b_dx4;                                                         \
1483                                                                                \
1484   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1485   vdup.u32 dx8, uv_dx8[0];                                                     \
1486                                                                                \
1487   vadd.u32 u_block, u_block, dx8;                                              \
1488   vdup.u32 dx8, uv_dx8[1];                                                     \
1489                                                                                \
1490   vadd.u32 v_block, v_block, dx8;                                              \
1491   vdup.u32 dx8, rg_dx8[0];                                                     \
1492                                                                                \
1493   vadd.u32 r_block, r_block, dx8;                                              \
1494   vdup.u32 dx8, rg_dx8[1];                                                     \
1495                                                                                \
1496   vadd.u32 g_block, g_block, dx8;                                              \
1497   vdup.u32 dx8, b_dx8;                                                         \
1498                                                                                \
1499   vadd.u32 b_block, b_block, dx8;                                              \
1500   vmovn.u16 u_whole_8, u_whole;                                                \
1501                                                                                \
1502   add fb_ptr, fb_ptr, #16;                                                     \
1503   vmovn.u16 v_whole_8, v_whole;                                                \
1504                                                                                \
1505   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1506   vmovn.u16 b_whole_8, b_whole;                                                \
1507                                                                                \
1508   pld [ fb_ptr ];                                                              \
1509                                                                                \
1510   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1511   subs span_num_blocks, span_num_blocks, #1;                                   \
1512                                                                                \
1513   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1514   setup_blocks_texture_##swizzling();                                          \
1515                                                                                \
1516   vmovn.u16 r_whole_8, r_whole;                                                \
1517   bne 4b;                                                                      \
1518                                                                                \
1519  5:                                                                            \
1520   vmovn.u16 g_whole_8, g_whole;                                                \
1521   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1522                                                                                \
1523   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1524   vdup.u8 draw_mask, right_mask;                                               \
1525                                                                                \
1526   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1527   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1528   vzip.u8 u_whole_8, v_whole_8;                                                \
1529                                                                                \
1530   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1531   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1532   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1533   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1534   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1535                                                                                \
1536  1:                                                                            \
1537   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1538   add span_b_offset, span_b_offset, #4;                                        \
1539                                                                                \
1540   add span_edge_data, span_edge_data, #8;                                      \
1541   subs num_spans, num_spans, #1;                                               \
1542                                                                                \
1543   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1544   bne 0b;                                                                      \
1545                                                                                \
1546   ldmia sp!, { r4 - r11, pc };                                                 \
1547                                                                                \
1548  2:                                                                            \
1549   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1550   vpush { texture_mask };                                                      \
1551   vpush { uvrg_dx4 };                                                          \
1552                                                                                \
1553   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1554   bl flush_render_block_buffer;                                                \
1555   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1556                                                                                \
1557   vpop { uvrg_dx4 };                                                           \
1558   vpop { texture_mask };                                                       \
1559                                                                                \
1560   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1561   vmov.u8 fb_mask_ptrs, #0;                                                    \
1562                                                                                \
1563   mov num_blocks, span_num_blocks;                                             \
1564   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1565   bal 3b                                                                       \
1566
1567
1568 setup_blocks_shaded_textured_builder(swizzled)
1569 setup_blocks_shaded_textured_builder(unswizzled)
1570
1571
1572 #define setup_blocks_unshaded_textured_builder(swizzling)                      \
1573 .align 3;                                                                      \
1574                                                                                \
1575 function(setup_blocks_unshaded_textured_dithered_##swizzling##_indirect)       \
1576   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1577   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1578                                                                                \
1579   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1580   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1581                                                                                \
1582   cmp num_spans, #0;                                                           \
1583   bxeq lr;                                                                     \
1584                                                                                \
1585   stmdb sp!, { r4 - r11, r14 };                                                \
1586   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1587                                                                                \
1588   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1589                                                                                \
1590   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1591   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1592                                                                                \
1593   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1594   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1595                                                                                \
1596   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1597                                                                                \
1598   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1599                                                                                \
1600  0:                                                                            \
1601   vmov.u8 fb_mask_ptrs, #0;                                                    \
1602                                                                                \
1603   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1604   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1605                                                                                \
1606   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1607   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1608                                                                                \
1609   cmp span_num_blocks, #0;                                                     \
1610   beq 1f;                                                                      \
1611                                                                                \
1612   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1613   add num_blocks, span_num_blocks, num_blocks;                                 \
1614                                                                                \
1615   cmp num_blocks, #MAX_BLOCKS;                                                 \
1616   bgt 2f;                                                                      \
1617                                                                                \
1618  3:                                                                            \
1619   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1620                                                                                \
1621   vdup.u32 v_left_x, left_x;                                                   \
1622   and y, y, #0x3;                                                              \
1623                                                                                \
1624   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1625   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1626                                                                                \
1627   and dither_shift, left_x, #0x03;                                             \
1628                                                                                \
1629   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1630   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1631                                                                                \
1632   mov dither_shift, dither_shift, lsl #3;                                      \
1633   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1634                                                                                \
1635   mov c_32, #32;                                                               \
1636   subs span_num_blocks, span_num_blocks, #1;                                   \
1637                                                                                \
1638   mov dither_row, dither_row, ror dither_shift;                                \
1639                                                                                \
1640   vdup.u32 dither_offsets_short, dither_row;                                   \
1641   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1642                                                                                \
1643   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1644                                                                                \
1645   vdup.u32 u_block, uv[0];                                                     \
1646                                                                                \
1647   vdup.u32 v_block, uv[1];                                                     \
1648   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1649                                                                                \
1650   vadd.u32 u_block, u_block, block_span;                                       \
1651   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1652                                                                                \
1653   vadd.u32 v_block, v_block, block_span;                                       \
1654   add block_ptr_b, block_ptr_a, #16;                                           \
1655                                                                                \
1656   vshrn.u32 u_whole_low, u_block, #16;                                         \
1657   vshrn.u32 v_whole_low, v_block, #16;                                         \
1658                                                                                \
1659   vdup.u32 dx4, uv_dx4[0];                                                     \
1660                                                                                \
1661   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1662   vdup.u32 dx4, uv_dx4[1];                                                     \
1663                                                                                \
1664   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1665   vdup.u32 dx8, uv_dx8[0];                                                     \
1666                                                                                \
1667   vadd.u32 u_block, u_block, dx8;                                              \
1668   vdup.u32 dx8, uv_dx8[1];                                                     \
1669                                                                                \
1670   vadd.u32 v_block, v_block, dx8;                                              \
1671   vmovn.u16 u_whole_8, u_whole;                                                \
1672                                                                                \
1673   vmovn.u16 v_whole_8, v_whole;                                                \
1674                                                                                \
1675   pld [ fb_ptr ];                                                              \
1676   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1677                                                                                \
1678   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1679   setup_blocks_texture_##swizzling();                                          \
1680                                                                                \
1681   beq 5f;                                                                      \
1682                                                                                \
1683  4:                                                                            \
1684   vshrn.u32 u_whole_low, u_block, #16;                                         \
1685                                                                                \
1686   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1687   vshrn.u32 v_whole_low, v_block, #16;                                         \
1688                                                                                \
1689   add block_ptr_b, block_ptr_b, #32;                                           \
1690   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1691                                                                                \
1692   vdup.u32 dx4, uv_dx4[0];                                                     \
1693   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1694   vdup.u32 dx4, uv_dx4[1];                                                     \
1695                                                                                \
1696   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1697   vdup.u32 dx8, uv_dx8[0];                                                     \
1698                                                                                \
1699   vadd.u32 u_block, u_block, dx8;                                              \
1700   vdup.u32 dx8, uv_dx8[1];                                                     \
1701                                                                                \
1702   vadd.u32 v_block, v_block, dx8;                                              \
1703   vmovn.u16 u_whole_8, u_whole;                                                \
1704                                                                                \
1705   add fb_ptr, fb_ptr, #16;                                                     \
1706   vmovn.u16 v_whole_8, v_whole;                                                \
1707                                                                                \
1708   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1709   pld [ fb_ptr ];                                                              \
1710                                                                                \
1711   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1712   subs span_num_blocks, span_num_blocks, #1;                                   \
1713                                                                                \
1714   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1715   setup_blocks_texture_##swizzling();                                          \
1716                                                                                \
1717   bne 4b;                                                                      \
1718                                                                                \
1719  5:                                                                            \
1720   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1721                                                                                \
1722   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1723   vdup.u8 draw_mask, right_mask;                                               \
1724                                                                                \
1725   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1726   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1727   vzip.u8 u_whole_8, v_whole_8;                                                \
1728                                                                                \
1729   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1730   add block_ptr_b, block_ptr_b, #32;                                           \
1731   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1732   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1733   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1734                                                                                \
1735  1:                                                                            \
1736   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1737   add span_edge_data, span_edge_data, #8;                                      \
1738   subs num_spans, num_spans, #1;                                               \
1739                                                                                \
1740   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1741   bne 0b;                                                                      \
1742                                                                                \
1743   ldmia sp!, { r4 - r11, pc };                                                 \
1744                                                                                \
1745  2:                                                                            \
1746   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1747   vpush { texture_mask };                                                      \
1748   vpush { uvrg_dx4 };                                                          \
1749                                                                                \
1750   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1751   bl flush_render_block_buffer;                                                \
1752   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1753                                                                                \
1754   vpop { uvrg_dx4 };                                                           \
1755   vpop { texture_mask };                                                       \
1756                                                                                \
1757   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1758   vmov.u8 fb_mask_ptrs, #0;                                                    \
1759                                                                                \
1760   mov num_blocks, span_num_blocks;                                             \
1761   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1762   bal 3b                                                                       \
1763
1764
1765 setup_blocks_unshaded_textured_builder(swizzled)
1766 setup_blocks_unshaded_textured_builder(unswizzled)
1767
1768
1769 .align 3
1770
1771 function(setup_blocks_unshaded_untextured_undithered_unswizzled_indirect)
1772   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1773   veor.u32 draw_mask, draw_mask, draw_mask
1774
1775   cmp num_spans, #0
1776   bxeq lr
1777
1778   stmdb sp!, { r4 - r11, r14 }
1779   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1780
1781   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1782
1783   ubfx color_r, color, #3, #5
1784   ubfx color_g, color, #11, #5
1785   ubfx color_b, color, #19, #5
1786
1787   orr color, color_r, color_b, lsl #10
1788   orr color, color, color_g, lsl #5
1789
1790   vdup.u16 colors, color
1791
1792   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1793   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1794
1795   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1796   add block_ptr_a, block_ptr_a, num_blocks, lsl #6
1797
1798  0:
1799   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1800   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1801
1802   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1803
1804   cmp span_num_blocks, #0
1805   beq 1f
1806
1807   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1808   add num_blocks, span_num_blocks, num_blocks
1809
1810   cmp num_blocks, #MAX_BLOCKS
1811   bgt 2f
1812
1813  3:
1814   add fb_ptr, fb_ptr, y, lsl #11
1815   and y, y, #0x3
1816
1817   add fb_ptr, fb_ptr, left_x, lsl #1
1818   mov c_32, #32
1819
1820   subs span_num_blocks, span_num_blocks, #1
1821
1822   add block_ptr_b, block_ptr_a, #16
1823   pld [ fb_ptr ]
1824
1825   vmov.u32 fb_mask_ptrs[1], fb_ptr
1826   beq 5f
1827
1828  4:
1829   vst1.u32 { draw_mask }, [ block_ptr_a, :128 ], c_32
1830   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1831   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1832
1833   add fb_ptr, fb_ptr, #16
1834   add block_ptr_b, block_ptr_b, #32
1835
1836   pld [ fb_ptr ]
1837
1838   vmov.u32 fb_mask_ptrs[1], fb_ptr
1839   subs span_num_blocks, span_num_blocks, #1
1840
1841   bne 4b
1842
1843  5:
1844   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1845
1846   vdup.u8 draw_mask_edge, right_mask
1847   vtst.u16 draw_mask_edge, draw_mask_edge, test_mask
1848
1849   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1850   vst1.u32 { draw_mask_edge }, [ block_ptr_a, :128 ], c_32
1851   add block_ptr_b, block_ptr_b, #32
1852   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1853
1854  1:
1855   add span_edge_data, span_edge_data, #8
1856   subs num_spans, num_spans, #1
1857
1858   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1859   bne 0b
1860
1861   ldmia sp!, { r4 - r11, pc }
1862                                                                            
1863  2:
1864   vpush { colors }
1865
1866   stmdb sp!, { r0 - r3, r12, r14 }
1867   bl flush_render_block_buffer
1868   ldmia sp!, { r0 - r3, r12, r14 }
1869
1870   vpop { colors }
1871
1872   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1873   veor.u32 draw_mask, draw_mask, draw_mask
1874
1875   mov num_blocks, span_num_blocks
1876   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1877   bal 3b
1878
1879
1880 #define mask_msb_scalar                                   r14
1881
1882 #define msb_mask                                          q15
1883
1884 #define pixels_low                                        d16
1885
1886 #define msb_mask_low                                      d30
1887 #define msb_mask_high                                     d31
1888
1889
1890 .align 3
1891
1892 function(setup_blocks_unshaded_untextured_undithered_unswizzled_direct)
1893   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1894
1895   cmp num_spans, #0
1896   bxeq lr
1897
1898   stmdb sp!, { r4 - r11, r14 }
1899
1900   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1901
1902   ubfx color_r, color, #3, #5
1903   ubfx color_g, color, #11, #5
1904
1905   ldrh mask_msb_scalar, [ psx_gpu, #psx_gpu_mask_msb_offset ]
1906   ubfx color_b, color, #19, #5
1907
1908   orr color, color_r, color_b, lsl #10
1909   orr color, color, color_g, lsl #5
1910   orr color, color, mask_msb_scalar
1911
1912   vdup.u16 colors, color
1913
1914   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1915   orr color, color, lsl #16
1916
1917
1918  0:
1919   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1920   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1921
1922   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1923
1924   cmp span_num_blocks, #0
1925   beq 1f
1926
1927   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1928
1929   add fb_ptr, fb_ptr, y, lsl #11
1930   subs span_num_blocks, span_num_blocks, #1
1931
1932   add fb_ptr, fb_ptr, left_x, lsl #1
1933   beq 3f
1934
1935  2:
1936   vst1.u32 { colors }, [ fb_ptr ]!
1937   subs span_num_blocks, span_num_blocks, #1
1938
1939   bne 2b
1940
1941  3:
1942   ldrb right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1943
1944   cmp right_mask, #0x0
1945   beq 5f
1946
1947   tst right_mask, #0xF
1948   streq color, [ fb_ptr ], #4
1949   moveq right_mask, right_mask, lsr #4
1950   streq color, [ fb_ptr ], #4
1951
1952   tst right_mask, #0x3
1953   streq color, [ fb_ptr ], #4
1954   moveq right_mask, right_mask, lsr #2
1955
1956   tst right_mask, #0x1
1957   streqh color, [ fb_ptr ]
1958
1959  1:
1960   add span_edge_data, span_edge_data, #8
1961   subs num_spans, num_spans, #1
1962   bne 0b
1963
1964   ldmia sp!, { r4 - r11, pc }
1965                                                                            
1966  5:
1967   vst1.u32 { colors }, [ fb_ptr ]
1968   bal 1b
1969
1970
1971 #undef c_64
1972
1973 #define c_64                                              r7
1974 #define rg_dx_ptr                                         r2
1975
1976
1977 #undef r_block
1978 #undef g_block
1979 #undef b_block
1980 #undef r_whole
1981 #undef g_whole
1982 #undef b_whole
1983 #undef r_whole_low
1984 #undef r_whole_high
1985 #undef g_whole_low
1986 #undef g_whole_high
1987 #undef b_whole_low
1988 #undef b_whole_high
1989 #undef r_whole_8
1990 #undef g_whole_8
1991 #undef b_whole_8
1992 #undef dither_offsets
1993 #undef rg_dx4
1994 #undef rg_dx8
1995 #undef dx4
1996 #undef dx8
1997 #undef v_left_x
1998 #undef uvrg
1999 #undef block_span
2000 #undef rg
2001 #undef draw_mask
2002 #undef test_mask
2003
2004 #define r_block                                           q0
2005 #define g_block                                           q1
2006 #define b_block                                           q2
2007
2008 #define r_whole                                           q3
2009 #define g_whole                                           q4
2010 #define b_whole                                           q5
2011
2012 #define r_whole_low                                       d6
2013 #define r_whole_high                                      d7
2014 #define g_whole_low                                       d8
2015 #define g_whole_high                                      d9
2016 #define b_whole_low                                       d10
2017 #define b_whole_high                                      d11
2018
2019 #define gb_whole_8                                        q6
2020
2021 #define g_whole_8                                         d12
2022 #define b_whole_8                                         d13
2023
2024 #define r_whole_8                                         d14
2025
2026 #define pixels                                            q8
2027
2028 #define rg_dx4                                            d18
2029 #define rg_dx8                                            d19
2030
2031 #define dx4                                               q10
2032 #define dx8                                               q10
2033
2034 #define v_left_x                                          d6
2035 #define uvrg                                              q4
2036 #define block_span                                        q5
2037
2038 #define rg                                                d9
2039
2040 #define d64_1                                             d22
2041 #define d64_128                                           d23
2042
2043 #define d128_4                                            q12
2044 #define d128_0x7                                          q13
2045
2046 #define d64_4                                             d24
2047
2048 #define dither_offsets                                    q14
2049 #define draw_mask                                         q15
2050
2051 #define dither_offsets_low                                d28
2052
2053 #define rg_dx                                             d0
2054 #define test_mask                                         q10
2055
2056
2057 #define setup_blocks_shaded_untextured_dither_a_dithered()                     \
2058   vqadd.u8 r_whole_8, r_whole_8, dither_offsets_low;                           \
2059   vqadd.u8 gb_whole_8, gb_whole_8, dither_offsets;                             \
2060
2061 #define setup_blocks_shaded_untextured_dither_b_dithered()                     \
2062   vqsub.u8 r_whole_8, r_whole_8, d64_4;                                        \
2063   vqsub.u8 gb_whole_8, gb_whole_8, d128_4                                      \
2064
2065 #define setup_blocks_shaded_untextured_dither_a_undithered()                   \
2066
2067 #define setup_blocks_shaded_untextured_dither_b_undithered()                   \
2068
2069
2070 #define setup_blocks_shaded_untextured_indirect_builder(dithering)             \
2071 .align 3;                                                                      \
2072                                                                                \
2073 function(setup_blocks_shaded_untextured_##dithering##_unswizzled_indirect)     \
2074   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
2075   add rg_dx_ptr, psx_gpu, #(psx_gpu_uvrg_dx_offset + 8);                       \
2076                                                                                \
2077   vld1.u32 { rg_dx }, [ rg_dx_ptr, :64 ];                                      \
2078                                                                                \
2079   cmp num_spans, #0;                                                           \
2080   bxeq lr;                                                                     \
2081                                                                                \
2082   stmdb sp!, { r4 - r11, r14 };                                                \
2083   vshl.u32 rg_dx4, rg_dx, #2;                                                  \
2084                                                                                \
2085   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
2086   vshl.u32 rg_dx8, rg_dx, #3;                                                  \
2087                                                                                \
2088   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
2089                                                                                \
2090   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
2091   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
2092                                                                                \
2093   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
2094   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
2095                                                                                \
2096   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
2097   vmov.u8 d64_1, #1;                                                           \
2098                                                                                \
2099   vmov.u8 d128_4, #4;                                                          \
2100   vmov.u8 d64_128, #128;                                                       \
2101                                                                                \
2102   vmov.u8 d128_0x7, #0x7;                                                      \
2103                                                                                \
2104  0:                                                                            \
2105   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
2106   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
2107                                                                                \
2108   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
2109   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
2110                                                                                \
2111   cmp span_num_blocks, #0;                                                     \
2112   beq 1f;                                                                      \
2113                                                                                \
2114   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
2115   add num_blocks, span_num_blocks, num_blocks;                                 \
2116                                                                                \
2117   cmp num_blocks, #MAX_BLOCKS;                                                 \
2118   bgt 2f;                                                                      \
2119                                                                                \
2120  3:                                                                            \
2121   ldr b, [ span_b_offset ];                                                    \
2122   add fb_ptr, fb_ptr, y, lsl #11;                                              \
2123                                                                                \
2124   vdup.u32 v_left_x, left_x;                                                   \
2125   and y, y, #0x3;                                                              \
2126                                                                                \
2127   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
2128   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
2129                                                                                \
2130   mla b, b_dx, left_x, b;                                                      \
2131   and dither_shift, left_x, #0x03;                                             \
2132                                                                                \
2133   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
2134   vshr.u32 rg_dx, rg_dx4, #2;                                                  \
2135                                                                                \
2136   mov dither_shift, dither_shift, lsl #3;                                      \
2137   vmla.u32 rg, rg_dx, v_left_x;                                                \
2138                                                                                \
2139   mov c_64, #64;                                                               \
2140   subs span_num_blocks, span_num_blocks, #1;                                   \
2141                                                                                \
2142   mov dither_row, dither_row, ror dither_shift;                                \
2143   mov b_dx4, b_dx, lsl #2;                                                     \
2144                                                                                \
2145   vdup.u32 dither_offsets, dither_row;                                         \
2146   add block_span_ptr, psx_gpu, #psx_gpu_r_block_span_offset;                   \
2147                                                                                \
2148   vdup.u32 b_block, b;                                                         \
2149   vadd.u8 dither_offsets, dither_offsets, d128_4;                              \
2150                                                                                \
2151   mov b_dx8, b_dx, lsl #3;                                                     \
2152   vdup.u32 r_block, rg[0];                                                     \
2153   vdup.u32 g_block, rg[1];                                                     \
2154                                                                                \
2155   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2156                                                                                \
2157   vadd.u32 r_block, r_block, block_span;                                       \
2158   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2159                                                                                \
2160   vadd.u32 g_block, g_block, block_span;                                       \
2161   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
2162                                                                                \
2163   vadd.u32 b_block, b_block, block_span;                                       \
2164   add block_ptr_b, block_ptr_a, #16;                                           \
2165                                                                                \
2166   vshrn.u32 r_whole_low, r_block, #16;                                         \
2167   vshrn.u32 g_whole_low, g_block, #16;                                         \
2168   vshrn.u32 b_whole_low, b_block, #16;                                         \
2169   vdup.u32 dx4, rg_dx4[0];                                                     \
2170                                                                                \
2171   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2172   vdup.u32 dx4, rg_dx4[1];                                                     \
2173                                                                                \
2174   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2175   vdup.u32 dx4, b_dx4;                                                         \
2176                                                                                \
2177   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2178   vdup.u32 dx8, rg_dx8[0];                                                     \
2179                                                                                \
2180   vadd.u32 r_block, r_block, dx8;                                              \
2181   vdup.u32 dx8, rg_dx8[1];                                                     \
2182                                                                                \
2183   vadd.u32 g_block, g_block, dx8;                                              \
2184   vdup.u32 dx8, b_dx8;                                                         \
2185                                                                                \
2186   vadd.u32 b_block, b_block, dx8;                                              \
2187                                                                                \
2188   vmovn.u16 r_whole_8, r_whole;                                                \
2189   vmovn.u16 g_whole_8, g_whole;                                                \
2190   vmovn.u16 b_whole_8, b_whole;                                                \
2191                                                                                \
2192   beq 5f;                                                                      \
2193   veor.u32 draw_mask, draw_mask, draw_mask;                                    \
2194                                                                                \
2195  4:                                                                            \
2196   setup_blocks_shaded_untextured_dither_a_##dithering();                       \
2197   vshrn.u32 r_whole_low, r_block, #16;                                         \
2198                                                                                \
2199   setup_blocks_shaded_untextured_dither_b_##dithering();                       \
2200   vshrn.u32 g_whole_low, g_block, #16;                                         \
2201                                                                                \
2202   vshrn.u32 b_whole_low, b_block, #16;                                         \
2203   str fb_ptr, [ block_ptr_a, #44 ];                                            \
2204                                                                                \
2205   vdup.u32 dx4, rg_dx4[0];                                                     \
2206   vshr.u8 r_whole_8, r_whole_8, #3;                                            \
2207   vbic.u8 gb_whole_8, gb_whole_8, d128_0x7;                                    \
2208                                                                                \
2209   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2210   vdup.u32 dx4, rg_dx4[1];                                                     \
2211                                                                                \
2212   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2213   vdup.u32 dx4, b_dx4;                                                         \
2214                                                                                \
2215   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2216   vdup.u32 dx8, rg_dx8[0];                                                     \
2217                                                                                \
2218   vmull.u8 pixels, r_whole_8, d64_1;                                           \
2219   vmlal.u8 pixels, g_whole_8, d64_4;                                           \
2220   vmlal.u8 pixels, b_whole_8, d64_128;                                         \
2221                                                                                \
2222   vadd.u32 r_block, r_block, dx8;                                              \
2223   vdup.u32 dx8, rg_dx8[1];                                                     \
2224                                                                                \
2225   vadd.u32 g_block, g_block, dx8;