psx_gpu: use different uvrgb phase for enhancement
[pcsx_rearmed.git] / plugins / gpu_neon / psx_gpu / psx_gpu_arm_neon.S
1 /*
2  * Copyright (C) 2011 Gilead Kutnick "Exophase" <exophase@gmail.com>
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of
7  * the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  */
14
15 #define MAX_SPANS                                         512
16 #define MAX_BLOCKS                                        64
17 #define MAX_BLOCKS_PER_ROW                                128
18
19 #include "psx_gpu_offsets.h"
20
21 #define psx_gpu_b_dx_offset (psx_gpu_b_block_span_offset + 4)
22
23 #define edge_data_left_x_offset                           0
24 #define edge_data_num_blocks_offset                       2
25 #define edge_data_right_mask_offset                       4
26 #define edge_data_y_offset                                6
27
28
29 #define psx_gpu                                           r0
30 #define v_a                                               r1
31 #define v_b                                               r2
32 #define v_c                                               r3
33
34 #define x0                                                r4
35 #define x1                                                r5
36 #define x2                                                r6
37 #define x0_x1                                             r5
38 #define x1_x2                                             r6
39 #define y0                                                r7
40 #define y1                                                r8
41 #define y2                                                r9
42 #define y0_y1                                             r7
43 #define y1_y2                                             r8
44 #define b0                                                r9
45 #define b1                                                r10
46 #define b2                                                r11
47 #define b0_b1                                             r10
48 #define b1_b2                                             r11
49
50
51 #define area_r_s                                          r5
52
53 #define g_bx0                                             r2
54 #define g_bx                                              r3
55 #define g_bx2                                             r4
56 #define g_bx3                                             r5
57 #define b_base                                            r6
58 #define g_by                                              r8
59
60 #define gs_bx                                             r7
61 #define gs_by                                             r10
62
63 #define ga_bx                                             g_bx
64 #define ga_by                                             g_by
65
66 #define gw_bx_h                                           g_bx
67 #define gw_by_h                                           g_by
68
69 #define gw_bx_l                                           r11
70 #define gw_by_l                                           gw_bx_l
71
72 #define store_a                                           r0
73 #define store_b                                           r1
74 #define store_inc                                         r5
75
76
77 #define v0                                                q0
78 #define uvrgb0                                            d0
79 #define x0_y0                                             d1
80
81 #define v1                                                q1
82 #define uvrgb1                                            d2
83 #define x1_y1                                             d3
84
85 #define v2                                                q2
86 #define uvrgb2                                            d4
87 #define x2_y2                                             d5
88
89 #define x0_ab                                             q3
90 #define uvrg_xxxx0                                        q3
91 #define uvrg0                                             d6
92 #define xxxx0                                             d7
93
94 #define x1_ab                                             q4
95 #define uvrg_xxxx1                                        q4
96 #define uvrg1                                             d8
97 #define xxxx1                                             d9
98
99 #define x2_ab                                             q5
100 #define uvrg_xxxx2                                        q5
101 #define uvrg2                                             d10
102 #define xxxx2                                             d11
103
104 #define y0_ab                                             q6
105 #define yyyy_uvrg0                                        q6
106 #define yyyy0                                             d12
107 #define uvrg0b                                            d13
108
109 #define y1_ab                                             q7
110 #define yyyy_uvrg1                                        q7
111 #define yyyy1                                             d14
112 #define uvrg1b                                            d15
113
114 #define y2_ab                                             q8
115 #define yyyy_uvrg2                                        q8
116 #define yyyy2                                             d16
117 #define uvrg2b                                            d17
118
119 #define d0_ab                                             q9
120 #define d0_a                                              d18
121 #define d0_b                                              d19
122
123 #define d1_ab                                             q10
124 #define d1_a                                              d20
125 #define d1_b                                              d21
126
127 #define d2_ab                                             q11
128 #define d2_a                                              d22
129 #define d2_b                                              d23
130
131 #define d3_ab                                             q12
132 #define d3_a                                              d24
133 #define d3_b                                              d25
134
135 #define ga_uvrg_x                                         q1
136 #define ga_uvrg_y                                         q4
137
138 #define dx                                                x0_x1
139 #define dy                                                y0_y1
140 #define db                                                b0_b1
141
142 #define uvrg_base                                         q11
143
144 #define gs_uvrg_x                                         q5
145 #define gs_uvrg_y                                         q6
146
147 #define g_uvrg_x                                          q1
148 #define ga_uv_x                                           d2
149 #define g_uv_x                                            d2
150 #define ga_rg_x                                           d3
151 #define g_rg_x                                            d3
152
153 #define g_uvrg_y                                          q4
154 #define ga_uv_y                                           d8
155 #define g_uv_y                                            d8
156 #define ga_rg_y                                           d9
157 #define g_rg_y                                            d9
158
159 #define gw_uv_x                                           q1
160 #define gw_rg_x                                           q2
161 #define gw_uv_y                                           q4
162 #define gw_rg_y                                           q3
163
164 #define w_mask                                            q9
165 #define w_mask_l                                          d18
166
167 #define r_shift                                           q10
168
169 #define uvrg_dx0                                          q0
170 #define uvrg_dx0l                                         d0
171 #define uvrg_dx0h                                         d1
172
173 #define uvrg_dx1                                          q1
174 #define uvrg_dx1l                                         d2
175 #define uvrg_dx1h                                         d3
176
177 #define uvrg_dx2                                          q2
178 #define uvrg_dx2l                                         d4
179 #define uvrg_dx2h                                         d5
180
181 #define uvrg_dx3                                          q3
182 #define uvrg_dx3l                                         d6
183 #define uvrg_dx3h                                         d7
184
185 #define uvrgb_phase                                       q13
186
187 .align 4
188
189 /* FIXME: users of this should be in psx_gpu instead */
190 #ifndef __PIC__
191 #define load_pointer(register, pointer)                                        \
192   movw register, :lower16:pointer;                                             \
193   movt register, :upper16:pointer;                                             \
194
195 #else
196 #define load_pointer(register, pointer)                                        \
197   ldr  register, =pointer                                                      \
198
199 #endif
200
201 #define function(name)                                                         \
202   .global name;                                                                \
203   name:                                                                        \
204
205 @ r0: psx_gpu
206 @ r1: v_a
207 @ r2: v_b
208 @ r3: v_c
209
210 function(compute_all_gradients)
211   // First compute the triangle area reciprocal and shift. The division will
212   // happen concurrently with much of the work which follows.
213   @ r12 = psx_gpu->triangle_area
214   ldr r12, [ psx_gpu, #psx_gpu_triangle_area_offset ]
215   stmdb sp!, { r4 - r11, lr }
216
217   @ load exponent of 62 into upper half of double
218   movw r4, #0
219   clz r14, r12                       @ r14 = shift
220
221   movt r4, #((62 + 1023) << 4)
222   mov r12, r12, lsl r14              @ r12 = triangle_area_normalized
223
224   @ load area normalized into lower half of double
225   mov r5, r12, lsr #10
226   vmov.f64 d30, r5, r4               @ d30 = (1 << 62) + ta_n
227
228   movt r4, #((1022 + 31) << 4)
229   mov r5, r12, lsl #20
230
231   add r4, r4, r12, lsr #11
232   vmov.f64 d31, r5, r4
233
234   vdiv.f64 d30, d30, d31             @ d30 = ((1 << 62) + ta_n) / ta_n
235
236   // ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0)) =
237   // ( d0       *  d1      ) - ( d2       *  d3      ) =
238   // ( m0                  ) - ( m1                  ) = gradient
239
240   // This is split to do 12 elements at a time over three sets: a, b, and c.
241   // Technically we only need to do 10 elements (uvrgb_x and uvrgb_y), so
242   // two of the slots are unused.
243
244   // Inputs are all 16-bit signed. The m0/m1 results are 32-bit signed, as
245   // is g.
246
247   // First type is:  uvrg bxxx xxxx 
248   // Second type is: yyyy ybyy uvrg 
249   // Since x_a and y_c are the same the same variable is used for both. 
250
251   vld1.u32 { v0 }, [ v_a, : 128 ]    @ v0 = { uvrg0, b0, x0, y0 }
252   ldrsh x0, [ v_a, #8 ]              @ load x0
253
254   vld1.u32 { v1 }, [ v_b, : 128 ]    @ v1 = { uvrg1, b1, x1, y1}
255   ldrh x1, [ v_b, #8 ]               @ load x1
256
257   vld1.u32 { v2 }, [ v_c, : 128 ]    @ v2 = { uvrg2, b2, x2, y2 }
258   ldrh x2, [ v_c, #8 ]               @ load x2
259
260   vmovl.u8 uvrg_xxxx0, uvrgb0        @ uvrg_xxxx0 = { uv0, rg0, b0-, -- }
261   ldrh y0, [ v_a, #10 ]              @ load y0
262
263   vmovl.u8 uvrg_xxxx1, uvrgb1        @ uvrg_xxxx1 = { uv1, rg1, b1-, -- }
264   ldrh y1, [ v_b, #10 ]              @ load y1
265
266   vmovl.u8 uvrg_xxxx2, uvrgb2        @ uvrg_xxxx2 = { uv2, rg2, b2-, -- }
267   ldrh y2, [ v_c, #10 ]              @ load y2
268
269   vmov.u8 uvrg0b, uvrg0              @ uvrg0b = { uv0, rg0 }
270   vdup.u16 xxxx0, x0_y0[0]           @ xxxx0 = { xx0, xx0 }
271
272   orr x1_x2, x1, x2, lsl #16         @ x1_x2 = { x1, x2 }
273   pkhbt x0_x1, x0, x1, lsl #16       @ x0_x1 = { x0, x1 }
274
275   vmov.u8 uvrg1b, uvrg1              @ uvrg1b = { uv1, rg1 }
276   vdup.u16 xxxx1, x1_y1[0]           @ xxxx1 = { xx1, xx1 }
277
278   vmov.u8 uvrg2b, uvrg2              @ uvrg2b = { uv2, rg2 }
279   vdup.u16 xxxx2, x2_y2[0]           @ xxxx2 = { xx2, xx2 }
280
281   ldrb b2, [ v_c, #4 ]               @ load b2
282   orr y0_y1, y0, y1, lsl #16         @ y0_y1 = { y0, y1 }
283
284   ldrb b1, [ v_b, #4 ]               @ load b1
285   orr y1_y2, y1, y2, lsl #16         @ y1_y2 = { y1, y2 }
286
287   vdup.u16 yyyy0, x0_y0[1]           @ yyyy0 = { yy0, yy0 }
288   vsub.s16 d0_ab, x1_ab, x0_ab
289
290   ldrb b0, [ v_a, #4 ]               @ load b0
291   orr b1_b2, b1, b2, lsl #16         @ b1_b2 = { b1, b2 }
292
293   vdup.u16 yyyy1, x1_y1[1]           @ yyyy1 = { yy1, yy1 }
294   vsub.s16 d2_ab, x2_ab, x1_ab
295
296   vdup.u16 yyyy2, x2_y2[1]           @ yyyy2 = { yy2, yy2 }
297   vsub.s16 d1_ab, y2_ab, y1_ab
298
299   orr b0_b1, b0, b1, lsl #16         @ b1_b2 = { b1, b2 }
300   ssub16 dx, x1_x2, x0_x1            @ dx = { x1 - x0, x2 - x1 }
301
302   ssub16 dy, y1_y2, y0_y1            @ dy = { y1 - y0, y2 - y1 }
303   ssub16 db, b1_b2, b0_b1            @ db = { b1 - b0, b2 - b1 }
304
305   vsub.s16 d3_ab, y1_ab, y0_ab
306   smusdx ga_by, dx, db               @ ga_by = ((x1 - x0) * (b2 - b1)) -
307                                      @         ((x2 - X1) * (b1 - b0)) 
308   vmull.s16 ga_uvrg_x, d0_a, d1_a
309   smusdx ga_bx, db, dy               @ ga_bx = ((b1 - b0) * (y2 - y1)) -
310                                      @         ((b2 - b1) * (y1 - y0))
311   vmlsl.s16 ga_uvrg_x, d2_a, d3_a
312   movs gs_bx, ga_bx, asr #31
313
314   vmull.s16 ga_uvrg_y, d0_b, d1_b
315   rsbmi ga_bx, ga_bx, #0
316
317   @ r12 = psx_gpu->uvrgb_phase
318   ldr r12, [ psx_gpu, #psx_gpu_uvrgb_phase_offset ]
319
320   vmlsl.s16 ga_uvrg_y, d2_b, d3_b
321   movs gs_by, ga_by, asr #31
322
323   vshr.u64 d0, d30, #22
324   add b_base, r12, b0, lsl #16
325
326   vdup.u32 uvrgb_phase, r12
327
328   rsbmi ga_by, ga_by, #0
329   vclt.s32 gs_uvrg_x, ga_uvrg_x, #0  @ gs_uvrg_x = ga_uvrg_x < 0
330
331   @ r12 = psx_gpu->triangle_winding_offset
332   ldrb r12, [ psx_gpu, #psx_gpu_triangle_winding_offset ]
333   vclt.s32 gs_uvrg_y, ga_uvrg_y, #0  @ gs_uvrg_y = ga_uvrg_y < 0
334
335   rsb r12, r12, #0                   @ r12 = -(triangle->winding)
336
337   vdup.u32 w_mask, r12               @ w_mask = { -w, -w, -w, -w }
338   sub r14, r14, #(62 - 12)           @ r14 = shift - (62 - FIXED_BITS)
339
340   vshll.u16 uvrg_base, uvrg0, #16    @ uvrg_base = uvrg0 << 16
341   vdup.u32 r_shift, r14              @ r_shift = { shift, shift, shift, shift }
342
343   vadd.u32 uvrg_base, uvrgb_phase
344   vabs.s32 ga_uvrg_x, ga_uvrg_x      @ ga_uvrg_x = abs(ga_uvrg_x)
345
346   vmov area_r_s, s0                  @ area_r_s = triangle_reciprocal
347   vabs.s32 ga_uvrg_y, ga_uvrg_y      @ ga_uvrg_y = abs(ga_uvrg_y)
348
349   vmull.u32 gw_rg_x, ga_rg_x, d0[0]
350   vmull.u32 gw_uv_x, ga_uv_x, d0[0]
351   vmull.u32 gw_rg_y, ga_rg_y, d0[0]
352   vmull.u32 gw_uv_y, ga_uv_y, d0[0]
353
354   vshl.u64 gw_rg_x, gw_rg_x, r_shift
355   vshl.u64 gw_uv_x, gw_uv_x, r_shift
356   vshl.u64 gw_rg_y, gw_rg_y, r_shift
357   vshl.u64 gw_uv_y, gw_uv_y, r_shift
358
359   veor.u32 gs_uvrg_x, gs_uvrg_x, w_mask
360   vmovn.u64 g_uv_x, gw_uv_x
361
362   veor.u32 gs_uvrg_y, gs_uvrg_y, w_mask
363   vmovn.u64 g_rg_x, gw_rg_x
364
365   veor.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
366   vmovn.u64 g_uv_y, gw_uv_y
367
368   vsub.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
369   vmovn.u64 g_rg_y, gw_rg_y
370
371   veor.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
372   mov ga_bx, ga_bx, lsl #13
373
374   vsub.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
375   mov ga_by, ga_by, lsl #13
376
377   vdup.u32 x0_y0, x0
378   umull gw_bx_l, gw_bx_h, ga_bx, area_r_s
379
380   vshl.u32 g_uvrg_x, g_uvrg_x, #4
381   vshl.u32 g_uvrg_y, g_uvrg_y, #4
382
383   umull gw_by_l, gw_by_h, ga_by, area_r_s
384   vmls.s32 uvrg_base, ga_uvrg_x, x0_y0[0]
385
386   eor gs_bx, gs_bx, r12
387   vadd.u32 uvrg_dx2, uvrg_dx1, uvrg_dx1
388
389   veor.u32 uvrg_dx0, uvrg_dx0, uvrg_dx0
390   eor gs_by, gs_by, r12
391
392   rsb r11, r14, #0                   @ r11 = negative shift for scalar lsr
393   add store_a, psx_gpu, #psx_gpu_uvrg_offset
394
395   sub r11, r11, #(32 - 13)
396
397   add store_b, store_a, #16
398   mov store_inc, #32
399
400   vadd.u32 uvrg_dx3, uvrg_dx2, uvrg_dx1
401   vst1.u32 { uvrg_base }, [ store_a, : 128 ], store_inc
402
403   vst1.u32 { uvrg_dx1 }, [ store_b, : 128 ], store_inc
404   mov g_bx, gw_bx_h, lsr r11
405
406   vst1.u32 { g_uvrg_y }, [ store_a, : 128 ], store_inc
407   mov g_by, gw_by_h, lsr r11
408
409   vst4.u32 { uvrg_dx0l, uvrg_dx1l, uvrg_dx2l, uvrg_dx3l },                     \
410    [ store_b, : 128 ], store_inc
411   eor g_bx, g_bx, gs_bx
412
413   vst4.u32 { uvrg_dx0h, uvrg_dx1h, uvrg_dx2h, uvrg_dx3h },                     \
414    [ store_b, : 128 ], store_inc
415   sub g_bx, g_bx, gs_bx
416
417   lsl g_bx, g_bx, #4  
418   eor g_by, g_by, gs_by
419
420   mls b_base, g_bx, x0, b_base
421   sub g_by, g_by, gs_by
422
423   lsl g_by, g_by, #4
424   mov g_bx0, #0
425
426   add g_bx2, g_bx, g_bx
427   add g_bx3, g_bx, g_bx2
428
429   stmia store_b, { g_bx0, g_bx, g_bx2, g_bx3, b_base, g_by }
430
431   ldmia sp!, { r4 - r11, pc }
432
433
434 #define psx_gpu                                  r0
435 #define v_a                                      r1
436 #define v_b                                      r2
437 #define v_c                                      r3
438
439 #define temp                                     r14
440
441 #define x_a                                      r4
442 #define x_b                                      r5
443 #define x_c                                      r6
444 #define y_a                                      r1
445 #define y_b                                      r2
446 #define y_c                                      r3
447
448 #define height_minor_a                           r7
449 #define height_minor_b                           r8
450 #define height_major                             r9
451 #define height                                   r9
452
453 #define reciprocal_table_ptr                     r10
454
455 #define edge_alt_low                             r4
456 #define edge_alt_high                            r5
457 #define edge_dx_dy_alt                           r6
458 #define edge_shift_alt                           r10
459
460 #define edge_dx_dy_alt_low                       r4
461 #define edge_dx_dy_alt_high                      r5
462
463 #define span_edge_data                           r4
464 #define span_uvrg_offset                         r5
465 #define span_b_offset                            r6
466
467 #define clip                                     r14
468
469 #define b                                        r11
470 #define b_dy                                     r12
471
472
473 #define alternate_x                              q0
474 #define alternate_dx_dy                          q1
475 #define alternate_x_32                           q2
476
477 #define alternate_x_low                          d0
478 #define alternate_x_high                         d1
479 #define alternate_dx_dy_low                      d2
480 #define alternate_dx_dy_high                     d3
481 #define alternate_x_32_low                       d4
482 #define alternate_x_32_high                      d5
483
484 #define left_x                                   q3
485 #define right_x                                  q4
486 #define left_dx_dy                               q5
487 #define right_dx_dy                              q6
488 #define left_edge                                q7
489 #define right_edge                               q8
490
491 #define left_x_low                               d6
492 #define left_x_high                              d7
493 #define right_x_low                              d8
494 #define right_x_high                             d9
495 #define left_dx_dy_low                           d10
496 #define left_dx_dy_high                          d11
497 #define right_dx_dy_low                          d12
498 #define right_dx_dy_high                         d13
499 #define left_edge_low                            d14
500 #define left_edge_high                           d15
501 #define right_edge_low                           d16
502 #define right_edge_high                          d17
503
504 #define y_mid_point                              d18
505 #define c_0x0004                                 d19
506
507 #define left_right_x_16                          q11
508 #define span_shifts_y                            q12
509 #define c_0x0001                                 q13
510
511 #define span_shifts                              d24
512 #define y_x4                                     d25
513 #define c_0xFFFE                                 d26
514 #define c_0x0007                                 d27
515
516 #define left_right_x_16_low                      d22
517 #define left_right_x_16_high                     d23
518
519 #define uvrg                                     q14
520 #define uvrg_dy                                  q15
521
522 #define alternate_x_16                           d4
523
524 #define v_clip                                   q3
525 #define v_clip_low                               d6
526
527 #define right_x_32                               q10
528 #define left_x_32                                q11
529 #define alternate_select                         d24
530
531 #define right_x_32_low                           d20
532 #define right_x_32_high                          d21
533 #define left_x_32_low                            d22
534 #define left_x_32_high                           d23
535
536 #define edges_xy                                 q0
537 #define edges_dx_dy                              d2
538 #define edge_shifts                              d3
539 #define edge_shifts_64                           q2
540
541 #define edges_xy_left                            d0
542 #define edges_xy_right                           d1
543
544 #define height_reciprocals                       d6
545 #define heights                                  d7
546
547 #define widths                                   d8
548 #define c_0x01                                   d9
549 #define x_starts                                 d10
550 #define x_ends                                   d11
551
552 #define heights_b                                d12
553 #define edges_dx_dy_64                           q10
554
555 #define edges_dx_dy_64_left                      d20
556 #define edges_dx_dy_64_right                     d21
557
558
559 #define setup_spans_prologue()                                                 \
560   stmdb sp!, { r4 - r11, lr };                                                 \
561                                                                                \
562   ldrsh x_a, [ v_a, #8 ];                                                      \
563   ldrsh x_b, [ v_b, #8 ];                                                      \
564   ldrsh x_c, [ v_c, #8 ];                                                      \
565   ldrsh y_a, [ v_a, #10 ];                                                     \
566   ldrsh y_b, [ v_b, #10 ];                                                     \
567   ldrsh y_c, [ v_c, #10 ];                                                     \
568                                                                                \
569   add temp, psx_gpu, #psx_gpu_uvrg_offset;                                     \
570   vld1.32 { uvrg }, [ temp ];                                                  \
571   add temp, psx_gpu, #psx_gpu_uvrg_dy_offset;                                  \
572   vld1.32 { uvrg_dy }, [ temp ];                                               \
573   load_pointer(reciprocal_table_ptr, reciprocal_table);                        \
574                                                                                \
575   vmov.u32 c_0x01, #0x01                                                       \
576
577 #define setup_spans_load_b()                                                   \
578   ldr b, [ psx_gpu, #psx_gpu_b_offset ];                                       \
579   ldr b_dy, [ psx_gpu, #psx_gpu_b_dy_offset ]                                  \
580
581 #define setup_spans_prologue_b()                                               \
582   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
583   add temp, psx_gpu, #psx_gpu_viewport_start_x_offset;                         \
584                                                                                \
585   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
586   vmov.u16 c_0x0004, #0x0004;                                                  \
587                                                                                \
588   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
589   vmov.u16 c_0x0001, #0x0001;                                                  \
590                                                                                \
591   vld1.u16 { left_edge_low[], left_edge_high[] }, [ temp ];                    \
592   add temp, psx_gpu, #psx_gpu_viewport_end_x_offset;                           \
593                                                                                \
594   vld1.u16 { right_edge_low[], right_edge_high[] }, [ temp ];                  \
595   vadd.u16 right_edge, right_edge, c_0x0001;                                   \
596                                                                                \
597   vmov.u16 c_0x0007, #0x0007;                                                  \
598   vmvn.u16 c_0xFFFE, #0x0001                                                   \
599
600
601 #define compute_edge_delta_x2()                                                \
602   ldr temp, [ reciprocal_table_ptr, height, lsl #2 ];                          \
603                                                                                \
604   vdup.u32 heights, height;                                                    \
605   vsub.u32 widths, x_ends, x_starts;                                           \
606                                                                                \
607   vdup.u32 edge_shifts, temp;                                                  \
608   vsub.u32 heights_b, heights, c_0x01;                                         \
609   vshr.u32 height_reciprocals, edge_shifts, #10;                               \
610                                                                                \
611   vmla.s32 heights_b, x_starts, heights;                                       \
612   vbic.u16 edge_shifts, #0xE0;                                                 \
613   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
614   vmull.s32 edges_xy, heights_b, height_reciprocals                            \
615
616 #define width_alt                 r6
617 #define height_reciprocal_alt     r11
618 #define height_b_alt              r12
619
620 #define compute_edge_delta_x3(start_c, height_a, height_b)                     \
621   vmov.u32 heights, height_a, height_b;                                        \
622   ldr temp, [ reciprocal_table_ptr, height_a, lsl #2 ];                        \
623   vmov.u32 edge_shifts[0], temp;                                               \
624   ldr temp, [ reciprocal_table_ptr, height_b, lsl #2 ];                        \
625   vmov.u32 edge_shifts[1], temp;                                               \
626   ldr edge_shift_alt, [ reciprocal_table_ptr, height_minor_b, lsl #2 ];        \
627                                                                                \
628   vsub.u32 widths, x_ends, x_starts;                                           \
629   sub width_alt, x_c, start_c;                                                 \
630                                                                                \
631   vsub.u32 heights_b, heights, c_0x01;                                         \
632   sub height_b_alt, height_minor_b, #1;                                        \
633                                                                                \
634   vshr.u32 height_reciprocals, edge_shifts, #10;                               \
635   lsr height_reciprocal_alt, edge_shift_alt, #10;                              \
636                                                                                \
637   vmla.s32 heights_b, x_starts, heights;                                       \
638   mla height_b_alt, height_minor_b, start_c, height_b_alt;                     \
639                                                                                \
640   vbic.u16 edge_shifts, #0xE0;                                                 \
641   and edge_shift_alt, edge_shift_alt, #0x1F;                                   \
642                                                                                \
643   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
644   mul edge_dx_dy_alt, width_alt, height_reciprocal_alt;                        \
645                                                                                \
646   vmull.s32 edges_xy, heights_b, height_reciprocals;                           \
647   smull edge_alt_low, edge_alt_high, height_b_alt, height_reciprocal_alt       \
648
649
650 #define setup_spans_adjust_y_up()                                              \
651   vsub.u32 y_x4, y_x4, c_0x0004                                                \
652
653 #define setup_spans_adjust_y_down()                                            \
654   vadd.u32 y_x4, y_x4, c_0x0004                                                \
655
656 #define setup_spans_adjust_interpolants_up()                                   \
657   vsub.u32 uvrg, uvrg, uvrg_dy;                                                \
658   sub b, b, b_dy                                                               \
659
660 #define setup_spans_adjust_interpolants_down()                                 \
661   vadd.u32 uvrg, uvrg, uvrg_dy;                                                \
662   add b, b, b_dy                                                               \
663
664
665 #define setup_spans_clip_interpolants_increment()                              \
666   mla b, b_dy, clip, b;                                                        \
667   vmla.s32 uvrg, uvrg_dy, v_clip                                               \
668
669 #define setup_spans_clip_interpolants_decrement()                              \
670   mls b, b_dy, clip, b;                                                        \
671   vmls.s32 uvrg, uvrg_dy, v_clip                                               \
672
673 #define setup_spans_clip_alternate_yes()                                       \
674   smlal edge_alt_low, edge_alt_high, edge_dx_dy_alt, clip                      \
675
676 #define setup_spans_clip_alternate_no()                                        \
677
678 #define setup_spans_clip(direction, alternate_active)                          \
679   vdup.u32 v_clip, clip;                                                       \
680   setup_spans_clip_alternate_##alternate_active();                             \
681   setup_spans_clip_interpolants_##direction();                                 \
682   vmlal.s32 edges_xy, edges_dx_dy, v_clip_low                                  \
683
684
685 #define setup_spans_adjust_edges_alternate_no(left_index, right_index)         \
686   vmovl.s32 edge_shifts_64, edge_shifts;                                       \
687   vmovl.s32 edges_dx_dy_64, edges_dx_dy;                                       \
688                                                                                \
689   vshl.s64 edges_xy, edges_xy, edge_shifts_64;                                 \
690   vshl.s64 edges_dx_dy_64, edges_dx_dy_64, edge_shifts_64;                     \
691                                                                                \
692   vmov left_x_low, edges_xy_##left_index;                                      \
693   vmov right_x_low, edges_xy_##right_index;                                    \
694                                                                                \
695   vmov left_dx_dy_low, edges_dx_dy_64_##left_index;                            \
696   vmov left_dx_dy_high, edges_dx_dy_64_##left_index;                           \
697   vmov right_dx_dy_low, edges_dx_dy_64_##right_index;                          \
698   vmov right_dx_dy_high, edges_dx_dy_64_##right_index;                         \
699                                                                                \
700   vadd.u64 left_x_high, left_x_low, left_dx_dy_low;                            \
701   vadd.u64 right_x_high, right_x_low, right_dx_dy_low;                         \
702                                                                                \
703   vadd.u64 left_dx_dy, left_dx_dy, left_dx_dy;                                 \
704   vadd.u64 right_dx_dy, right_dx_dy, right_dx_dy                               \
705
706
707 #define setup_spans_adjust_edges_alternate_yes(left_index, right_index)        \
708   setup_spans_adjust_edges_alternate_no(left_index, right_index);              \
709                                                                                \
710   vdup.u16 y_mid_point, y_b;                                                   \
711   rsb temp, edge_shift_alt, #32;                                               \
712                                                                                \
713   lsl edge_alt_high, edge_alt_high, edge_shift_alt;                            \
714   orr edge_alt_high, edge_alt_high, edge_alt_low, lsr temp;                    \
715   lsl edge_alt_low, edge_alt_low, edge_shift_alt;                              \
716   vmov alternate_x_low, edge_alt_low, edge_alt_high;                           \
717                                                                                \
718   asr edge_dx_dy_alt_high, edge_dx_dy_alt, temp;                               \
719   lsl edge_dx_dy_alt_low, edge_dx_dy_alt, edge_shift_alt;                      \
720   vmov alternate_dx_dy_low, edge_dx_dy_alt_low, edge_dx_dy_alt_high;           \
721   vmov alternate_dx_dy_high, alternate_dx_dy_low;                              \
722                                                                                \
723   vadd.u64 alternate_x_high, alternate_x_low, alternate_dx_dy_low;             \
724   vadd.u64 alternate_dx_dy, alternate_dx_dy, alternate_dx_dy                   \
725
726
727 #define setup_spans_y_select_up()                                              \
728   vclt.s16 alternate_select, y_x4, y_mid_point                                 \
729
730 #define setup_spans_y_select_down()                                            \
731   vcgt.s16 alternate_select, y_x4, y_mid_point                                 \
732
733
734 #define setup_spans_alternate_select_left()                                    \
735   vbit.u16 left_right_x_16_low, alternate_x_16, alternate_select               \
736
737 #define setup_spans_alternate_select_right()                                   \
738   vbit.u16 left_right_x_16_high, alternate_x_16, alternate_select              \
739
740
741 #define setup_spans_set_x4_alternate_yes(alternate, direction)                 \
742   vshrn.s64 alternate_x_32_low, alternate_x, #32;                              \
743   vshrn.s64 left_x_32_low, left_x, #32;                                        \
744   vshrn.s64 right_x_32_low, right_x, #32;                                      \
745                                                                                \
746   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
747   vadd.u64 left_x, left_x, left_dx_dy;                                         \
748   vadd.u64 right_x, right_x, right_dx_dy;                                      \
749                                                                                \
750   vshrn.s64 alternate_x_32_high, alternate_x, #32;                             \
751   vshrn.s64 left_x_32_high, left_x, #32;                                       \
752   vshrn.s64 right_x_32_high, right_x, #32;                                     \
753                                                                                \
754   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
755   vadd.u64 left_x, left_x, left_dx_dy;                                         \
756   vadd.u64 right_x, right_x, right_dx_dy;                                      \
757                                                                                \
758   vmovn.u32 alternate_x_16, alternate_x_32;                                    \
759   setup_spans_y_select_##direction();                                          \
760   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
761                                                                                \
762   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
763   setup_spans_alternate_select_##alternate();                                  \
764                                                                                \
765   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
766   str b, [ span_b_offset ], #4;                                                \
767   setup_spans_adjust_interpolants_##direction();                               \
768                                                                                \
769   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
770                                                                                \
771   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
772   str b, [ span_b_offset ], #4;                                                \
773   setup_spans_adjust_interpolants_##direction();                               \
774                                                                                \
775   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
776                                                                                \
777   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
778   str b, [ span_b_offset ], #4;                                                \
779   setup_spans_adjust_interpolants_##direction();                               \
780                                                                                \
781   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
782   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
783   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
784                                                                                \
785   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
786   str b, [ span_b_offset ], #4;                                                \
787   setup_spans_adjust_interpolants_##direction();                               \
788                                                                                \
789   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
790   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
791                                                                                \
792   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
793                                                                                \
794   setup_spans_adjust_y_##direction()                                           \
795
796
797 #define setup_spans_set_x4_alternate_no(alternate, direction)                  \
798   vshrn.s64 left_x_32_low, left_x, #32;                                        \
799   vshrn.s64 right_x_32_low, right_x, #32;                                      \
800                                                                                \
801   vadd.u64 left_x, left_x, left_dx_dy;                                         \
802   vadd.u64 right_x, right_x, right_dx_dy;                                      \
803                                                                                \
804   vshrn.s64 left_x_32_high, left_x, #32;                                       \
805   vshrn.s64 right_x_32_high, right_x, #32;                                     \
806                                                                                \
807   vadd.u64 left_x, left_x, left_dx_dy;                                         \
808   vadd.u64 right_x, right_x, right_dx_dy;                                      \
809                                                                                \
810   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
811   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
812                                                                                \
813   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
814   str b, [ span_b_offset ], #4;                                                \
815   setup_spans_adjust_interpolants_##direction();                               \
816                                                                                \
817   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
818                                                                                \
819   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
820   str b, [ span_b_offset ], #4;                                                \
821   setup_spans_adjust_interpolants_##direction();                               \
822                                                                                \
823   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
824                                                                                \
825   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
826   str b, [ span_b_offset ], #4;                                                \
827   setup_spans_adjust_interpolants_##direction();                               \
828                                                                                \
829   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
830   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
831   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
832                                                                                \
833   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
834   str b, [ span_b_offset ], #4;                                                \
835   setup_spans_adjust_interpolants_##direction();                               \
836                                                                                \
837   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
838   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
839                                                                                \
840   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
841                                                                                \
842   setup_spans_adjust_y_##direction()                                           \
843
844
845 #define edge_adjust_low           r11
846 #define edge_adjust_high          r12
847
848 #define setup_spans_alternate_adjust_yes()                                     \
849   smull edge_adjust_low, edge_adjust_high, edge_dx_dy_alt, height_minor_a;     \
850   subs edge_alt_low, edge_alt_low, edge_adjust_low;                            \
851   sbc edge_alt_high, edge_alt_high, edge_adjust_high                           \
852
853 #define setup_spans_alternate_adjust_no()                                      \
854
855
856 #define setup_spans_down(left_index, right_index, alternate, alternate_active) \
857   setup_spans_alternate_adjust_##alternate_active();                           \
858   setup_spans_load_b();                                                        \
859                                                                                \
860   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                     \
861   subs y_c, y_c, temp;                                                         \
862   subgt height, height, y_c;                                                   \
863   addgt height, height, #1;                                                    \
864                                                                                \
865   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                   \
866   subs clip, temp, y_a;                                                        \
867   ble 0f;                                                                      \
868                                                                                \
869   sub height, height, clip;                                                    \
870   add y_a, y_a, clip;                                                          \
871   setup_spans_clip(increment, alternate_active);                               \
872                                                                                \
873  0:                                                                            \
874   cmp height, #0;                                                              \
875   ble 1f;                                                                      \
876                                                                                \
877   orr temp, y_a, y_a, lsl #16;                                                 \
878   add temp, temp, #(1 << 16);                                                  \
879   add y_a, temp, #2;                                                           \
880   add y_a, y_a, #(2 << 16);                                                    \
881   vmov.u32 y_x4, temp, y_a;                                                    \
882                                                                                \
883   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
884    right_index);                                                               \
885   setup_spans_prologue_b();                                                    \
886                                                                                \
887   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
888                                                                                \
889  2:                                                                            \
890   setup_spans_set_x4_alternate_##alternate_active(alternate, down);            \
891   subs height, height, #4;                                                     \
892   bhi 2b;                                                                      \
893                                                                                \
894  1:                                                                            \
895
896
897 #define setup_spans_alternate_pre_increment_yes()                              \
898   adds edge_alt_low, edge_alt_low, edge_dx_dy_alt;                             \
899   adc edge_alt_high, edge_alt_high, edge_dx_dy_alt, asr #31                    \
900
901 #define setup_spans_alternate_pre_increment_no()                               \
902
903
904 #define setup_spans_up_decrement_yes()                                         \
905   suble height, height, #1                                                     \
906
907 #define setup_spans_up_decrement_no()                                          \
908
909
910 #define setup_spans_up(left_index, right_index, alternate, alternate_active)   \
911   setup_spans_alternate_adjust_##alternate_active();                           \
912   setup_spans_load_b();                                                        \
913   sub y_a, y_a, #1;                                                            \
914                                                                                \
915   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                    \
916   subs temp, temp, y_c;                                                        \
917   subgt height, height, temp;                                                  \
918   setup_spans_up_decrement_##alternate_active();                               \
919                                                                                \
920   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                      \
921   subs clip, y_a, temp;                                                        \
922   ble 0f;                                                                      \
923                                                                                \
924   sub height, height, clip;                                                    \
925   sub y_a, y_a, clip;                                                          \
926   setup_spans_clip(decrement, alternate_active);                               \
927                                                                                \
928  0:                                                                            \
929   cmp height, #0;                                                              \
930   ble 1f;                                                                      \
931                                                                                \
932   orr temp, y_a, y_a, lsl #16;                                                 \
933   sub temp, temp, #(1 << 16);                                                  \
934   sub y_a, temp, #2;                                                           \
935   sub y_a, y_a, #(2 << 16);                                                    \
936   vmov.u32 y_x4, temp, y_a;                                                    \
937                                                                                \
938   vaddw.s32 edges_xy, edges_xy, edges_dx_dy;                                   \
939                                                                                \
940   setup_spans_alternate_pre_increment_##alternate_active();                    \
941   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
942    right_index);                                                               \
943   setup_spans_adjust_interpolants_up();                                        \
944   setup_spans_prologue_b();                                                    \
945                                                                                \
946   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
947                                                                                \
948  2:                                                                            \
949   setup_spans_set_x4_alternate_##alternate_active(alternate, up);              \
950   subs height, height, #4;                                                     \
951   bhi 2b;                                                                      \
952                                                                                \
953  1:                                                                            \
954
955
956 #define setup_spans_epilogue()                                                 \
957   ldmia sp!, { r4 - r11, pc }                                                  \
958
959
960 #define setup_spans_up_up(minor, major)                                        \
961   setup_spans_prologue();                                                      \
962   sub height_minor_a, y_a, y_b;                                                \
963   sub height_minor_b, y_b, y_c;                                                \
964   sub height, y_a, y_c;                                                        \
965                                                                                \
966   vdup.u32 x_starts, x_a;                                                      \
967   vmov.u32 x_ends, x_c, x_b;                                                   \
968                                                                                \
969   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
970   setup_spans_up(major, minor, minor, yes);                                    \
971   setup_spans_epilogue()                                                       \
972
973 function(setup_spans_up_left)
974   setup_spans_up_up(left, right)
975
976 function(setup_spans_up_right)
977   setup_spans_up_up(right, left)
978
979 .pool
980
981 #define setup_spans_down_down(minor, major)                                    \
982   setup_spans_prologue();                                                      \
983   sub height_minor_a, y_b, y_a;                                                \
984   sub height_minor_b, y_c, y_b;                                                \
985   sub height, y_c, y_a;                                                        \
986                                                                                \
987   vdup.u32 x_starts, x_a;                                                      \
988   vmov.u32 x_ends, x_c, x_b;                                                   \
989                                                                                \
990   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
991   setup_spans_down(major, minor, minor, yes);                                  \
992   setup_spans_epilogue()                                                       \
993
994 function(setup_spans_down_left)
995   setup_spans_down_down(left, right)
996
997 function(setup_spans_down_right)
998   setup_spans_down_down(right, left)
999
1000
1001 #define setup_spans_up_flat()                                                  \
1002   sub height, y_a, y_c;                                                        \
1003                                                                                \
1004   compute_edge_delta_x2();                                                     \
1005   setup_spans_up(left, right, none, no);                                       \
1006   setup_spans_epilogue()                                                       \
1007
1008 function(setup_spans_up_a)
1009   setup_spans_prologue()
1010
1011   vmov.u32 x_starts, x_a, x_b
1012   vdup.u32 x_ends, x_c
1013
1014   setup_spans_up_flat()
1015
1016 function(setup_spans_up_b)
1017   setup_spans_prologue()
1018
1019   vdup.u32 x_starts, x_a
1020   vmov.u32 x_ends, x_b, x_c
1021
1022   setup_spans_up_flat()
1023
1024 #define setup_spans_down_flat()                                                \
1025   sub height, y_c, y_a;                                                        \
1026                                                                                \
1027   compute_edge_delta_x2();                                                     \
1028   setup_spans_down(left, right, none, no);                                     \
1029   setup_spans_epilogue()                                                       \
1030
1031 function(setup_spans_down_a)
1032   setup_spans_prologue()
1033
1034   vmov.u32 x_starts, x_a, x_b
1035   vdup.u32 x_ends, x_c
1036
1037   setup_spans_down_flat()
1038
1039 function(setup_spans_down_b)
1040   setup_spans_prologue()
1041
1042   vdup.u32 x_starts, x_a
1043   vmov.u32 x_ends, x_b, x_c
1044
1045   setup_spans_down_flat()
1046
1047
1048 #define middle_y                                          r9
1049
1050 #define edges_xy_b                                        q11
1051 #define edges_dx_dy_b                                     d26
1052 #define edge_shifts_b                                     d27
1053 #define edges_dx_dy_and_shifts_b                          q13
1054 #define height_increment                                  d20
1055
1056 #define edges_dx_dy_and_shifts                            q1
1057
1058 #define edges_xy_b_left                                   d22
1059 #define edges_xy_b_right                                  d23
1060
1061 #define setup_spans_up_down_load_edge_set_b()                                  \
1062   vmov edges_xy, edges_xy_b;                                                   \
1063   vmov edges_dx_dy_and_shifts, edges_dx_dy_and_shifts_b                        \
1064
1065
1066 function(setup_spans_up_down)
1067   setup_spans_prologue()
1068
1069   // s32 middle_y = y_a;
1070   sub height_minor_a, y_a, y_b
1071   sub height_minor_b, y_c, y_a
1072   sub height_major, y_c, y_b
1073
1074   vmov.u32 x_starts, x_a, x_c
1075   vdup.u32 x_ends, x_b
1076
1077   compute_edge_delta_x3(x_a, height_minor_a, height_major)
1078
1079   mov temp, #0
1080   vmov.u32 height_increment, temp, height_minor_b
1081   vmlal.s32 edges_xy, edges_dx_dy, height_increment
1082
1083   vmov edges_xy_b_left, edge_alt_low, edge_alt_high
1084   vmov edges_xy_b_right, edges_xy_right
1085
1086   vmov edge_shifts_b, edge_shifts
1087   vmov.u32 edge_shifts_b[0], edge_shift_alt
1088
1089   vneg.s32 edges_dx_dy_b, edges_dx_dy
1090   vmov.u32 edges_dx_dy_b[0], edge_dx_dy_alt
1091
1092   mov middle_y, y_a
1093   
1094   setup_spans_load_b()
1095   sub y_a, y_a, #1
1096
1097   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1098   subs temp, temp, y_b
1099   subgt height_minor_a, height_minor_a, temp
1100
1101   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1102   subs clip, y_a, temp
1103   ble 0f
1104
1105   sub height_minor_a, height_minor_a, clip
1106   sub y_a, y_a, clip
1107   setup_spans_clip(decrement, no)
1108
1109  0:                                                                
1110   cmp height_minor_a, #0
1111   ble 3f
1112
1113   orr temp, y_a, y_a, lsl #16
1114   sub temp, temp, #(1 << 16)
1115   sub y_a, temp, #2
1116   sub y_a, y_a, #(2 << 16)
1117   vmov.u32 y_x4, temp, y_a
1118
1119   vaddw.s32 edges_xy, edges_xy, edges_dx_dy
1120
1121   strh height_minor_a, [ psx_gpu, #psx_gpu_num_spans_offset ]
1122
1123   setup_spans_adjust_edges_alternate_no(left, right); 
1124   setup_spans_adjust_interpolants_up()
1125   setup_spans_up_down_load_edge_set_b()
1126
1127   setup_spans_prologue_b()
1128
1129
1130  2: 
1131   setup_spans_set_x4_alternate_no(none, up)
1132   subs height_minor_a, height_minor_a, #4
1133   bhi 2b
1134
1135   add span_edge_data, span_edge_data, height_minor_a, lsl #3
1136   add span_uvrg_offset, span_uvrg_offset, height_minor_a, lsl #4
1137   add span_b_offset, span_b_offset, height_minor_a, lsl #2
1138
1139  4:
1140   add temp, psx_gpu, #psx_gpu_uvrg_offset
1141   vld1.32 { uvrg }, [ temp ]
1142   mov y_a, middle_y
1143   
1144   setup_spans_load_b()
1145
1146   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1147   subs y_c, y_c, temp
1148   subgt height_minor_b, height_minor_b, y_c
1149   addgt height_minor_b, height_minor_b, #1
1150
1151   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1152   subs clip, temp, y_a
1153   ble 0f
1154
1155   sub height_minor_b, height_minor_b, clip
1156   add y_a, y_a, clip
1157   setup_spans_clip(increment, no)
1158
1159  0:
1160   cmp height_minor_b, #0
1161   ble 1f
1162
1163   orr temp, y_a, y_a, lsl #16
1164   add temp, temp, #(1 << 16) 
1165   add y_a, temp, #2
1166   add y_a, y_a, #(2 << 16)
1167   vmov.u32 y_x4, temp, y_a
1168
1169   setup_spans_adjust_edges_alternate_no(left, right)
1170
1171   ldrh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1172   add temp, temp, height_minor_b
1173   strh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1174
1175  2:                                                     
1176   setup_spans_set_x4_alternate_no(none, down)
1177   subs height_minor_b, height_minor_b, #4
1178   bhi 2b
1179
1180  1:
1181   setup_spans_epilogue()
1182
1183  3:
1184   setup_spans_up_down_load_edge_set_b()
1185   setup_spans_prologue_b()
1186   bal 4b
1187
1188 .pool
1189
1190 #undef span_uvrg_offset
1191 #undef span_edge_data
1192 #undef span_b_offset
1193 #undef left_x
1194 #undef b
1195
1196 #define psx_gpu                                  r0
1197 #define num_spans                                r1
1198 #define span_uvrg_offset                         r2
1199 #define span_edge_data                           r3
1200 #define span_b_offset                            r4
1201 #define b_dx                                     r5
1202 #define span_num_blocks                          r6
1203 #define y                                        r7
1204 #define left_x                                   r8
1205 #define b                                        r9
1206 #define dither_offset_ptr                        r10
1207 #define block_ptr_a                              r11
1208 #define fb_ptr                                   r12
1209 #define num_blocks                               r14
1210
1211 #define uvrg_dx_ptr                              r2
1212 #define texture_mask_ptr                         r3
1213 #define dither_shift                             r8
1214 #define dither_row                               r10
1215
1216 #define c_32                                     r7
1217 #define b_dx4                                    r8
1218 #define b_dx8                                    r9
1219 #define block_ptr_b                              r10
1220
1221 #define block_span_ptr                           r10
1222 #define right_mask                               r8
1223
1224 #define color                                    r2
1225 #define color_r                                  r3
1226 #define color_g                                  r4
1227 #define color_b                                  r5
1228
1229 #undef uvrg
1230
1231 #define u_block                                  q0
1232 #define v_block                                  q1
1233 #define r_block                                  q2
1234 #define g_block                                  q3
1235 #define b_block                                  q4
1236
1237 #define uv_dx4                                   d10
1238 #define rg_dx4                                   d11
1239 #define uv_dx8                                   d12
1240 #define rg_dx8                                   d13
1241 #define b_whole_8                                d14
1242 #define fb_mask_ptrs                             d15
1243
1244 #define uvrg_dx4                                 q5
1245 #define uvrg_dx8                                 q6
1246 #define uv_dx8                                   d12
1247 #define rg_dx8                                   d13
1248
1249 #define u_whole                                  q8
1250 #define v_whole                                  q9
1251 #define r_whole                                  q10
1252 #define g_whole                                  q11
1253 #define b_whole                                  q12
1254
1255 #define u_whole_low                              d16
1256 #define u_whole_high                             d17
1257 #define v_whole_low                              d18
1258 #define v_whole_high                             d19
1259 #define r_whole_low                              d20
1260 #define r_whole_high                             d21
1261 #define g_whole_low                              d22
1262 #define g_whole_high                             d23
1263 #define b_whole_low                              d24
1264 #define b_whole_high                             d25
1265
1266 #define dx4                                      q13
1267 #define dx8                                      q13
1268
1269 #define u_whole_8                                d26
1270 #define v_whole_8                                d27
1271 #define u_whole_8b                               d24
1272 #define r_whole_8                                d24
1273 #define g_whole_8                                d25
1274
1275 #define uv_whole_8                               q13
1276 #define uv_whole_8b                              q14
1277
1278 #define dither_offsets                           q14
1279 #define texture_mask                             q15
1280 #define texture_mask_u                           d30
1281 #define texture_mask_v                           d31
1282
1283 #define dither_offsets_short                     d28
1284
1285 #define v_left_x                                 q8
1286 #define uvrg                                     q9
1287 #define block_span                               q10
1288
1289 #define uv                                       d18
1290 #define rg                                       d19
1291
1292 #define draw_mask                                q1
1293 #define draw_mask_edge                           q13
1294 #define test_mask                                q0
1295
1296 #define uvrg_dx                                  q3
1297
1298 #define colors                                   q2
1299
1300 #define setup_blocks_texture_swizzled()                                        \
1301   vand.u8 u_whole_8b, u_whole_8, texture_mask_u;                               \
1302   vsli.u8 u_whole_8, v_whole_8, #4;                                            \
1303   vsri.u8 v_whole_8, u_whole_8b, #4                                            \
1304
1305 #define setup_blocks_texture_unswizzled()                                      \
1306
1307
1308 #define setup_blocks_shaded_textured_builder(swizzling)                        \
1309 .align 3;                                                                      \
1310                                                                                \
1311 function(setup_blocks_shaded_textured_dithered_##swizzling##_indirect)         \
1312   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1313   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1314                                                                                \
1315   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1316   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1317                                                                                \
1318   cmp num_spans, #0;                                                           \
1319   bxeq lr;                                                                     \
1320                                                                                \
1321   stmdb sp!, { r4 - r11, r14 };                                                \
1322   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1323                                                                                \
1324   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
1325   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1326                                                                                \
1327   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1328   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1329                                                                                \
1330   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1331   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1332                                                                                \
1333   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
1334   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1335                                                                                \
1336   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1337                                                                                \
1338  0:                                                                            \
1339   vmov.u8 fb_mask_ptrs, #0;                                                    \
1340                                                                                \
1341   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1342   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1343                                                                                \
1344   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1345   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_out_ptr_offset ];                       \
1346                                                                                \
1347   cmp span_num_blocks, #0;                                                     \
1348   beq 1f;                                                                      \
1349                                                                                \
1350   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1351   add num_blocks, span_num_blocks, num_blocks;                                 \
1352                                                                                \
1353   cmp num_blocks, #MAX_BLOCKS;                                                 \
1354   bgt 2f;                                                                      \
1355                                                                                \
1356  3:                                                                            \
1357   ldr b, [ span_b_offset ];                                                    \
1358   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1359                                                                                \
1360   vdup.u32 v_left_x, left_x;                                                   \
1361   and y, y, #0x3;                                                              \
1362                                                                                \
1363   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1364   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1365                                                                                \
1366   mla b, b_dx, left_x, b;                                                      \
1367   and dither_shift, left_x, #0x03;                                             \
1368                                                                                \
1369   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1370   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1371                                                                                \
1372   mov dither_shift, dither_shift, lsl #3;                                      \
1373   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1374                                                                                \
1375   mov c_32, #32;                                                               \
1376   subs span_num_blocks, span_num_blocks, #1;                                   \
1377                                                                                \
1378   mov dither_row, dither_row, ror dither_shift;                                \
1379   mov b_dx4, b_dx, lsl #2;                                                     \
1380                                                                                \
1381   vdup.u32 dither_offsets_short, dither_row;                                   \
1382   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1383                                                                                \
1384   vdup.u32 b_block, b;                                                         \
1385   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1386                                                                                \
1387   vdup.u32 u_block, uv[0];                                                     \
1388   mov b_dx8, b_dx, lsl #3;                                                     \
1389                                                                                \
1390   vdup.u32 v_block, uv[1];                                                     \
1391   vdup.u32 r_block, rg[0];                                                     \
1392   vdup.u32 g_block, rg[1];                                                     \
1393                                                                                \
1394   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1395                                                                                \
1396   vadd.u32 u_block, u_block, block_span;                                       \
1397   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1398                                                                                \
1399   vadd.u32 v_block, v_block, block_span;                                       \
1400   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1401                                                                                \
1402   vadd.u32 r_block, r_block, block_span;                                       \
1403   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1404                                                                                \
1405   vadd.u32 g_block, g_block, block_span;                                       \
1406   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
1407                                                                                \
1408   vadd.u32 b_block, b_block, block_span;                                       \
1409   add block_ptr_b, block_ptr_a, #16;                                           \
1410                                                                                \
1411   vshrn.u32 u_whole_low, u_block, #16;                                         \
1412   vshrn.u32 v_whole_low, v_block, #16;                                         \
1413   vshrn.u32 r_whole_low, r_block, #16;                                         \
1414   vshrn.u32 g_whole_low, g_block, #16;                                         \
1415                                                                                \
1416   vdup.u32 dx4, uv_dx4[0];                                                     \
1417   vshrn.u32 b_whole_low, b_block, #16;                                         \
1418                                                                                \
1419   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1420   vdup.u32 dx4, uv_dx4[1];                                                     \
1421                                                                                \
1422   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1423   vdup.u32 dx4, rg_dx4[0];                                                     \
1424                                                                                \
1425   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1426   vdup.u32 dx4, rg_dx4[1];                                                     \
1427                                                                                \
1428   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1429   vdup.u32 dx4, b_dx4;                                                         \
1430                                                                                \
1431   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1432   vdup.u32 dx8, uv_dx8[0];                                                     \
1433                                                                                \
1434   vadd.u32 u_block, u_block, dx8;                                              \
1435   vdup.u32 dx8, uv_dx8[1];                                                     \
1436                                                                                \
1437   vadd.u32 v_block, v_block, dx8;                                              \
1438   vdup.u32 dx8, rg_dx8[0];                                                     \
1439                                                                                \
1440   vadd.u32 r_block, r_block, dx8;                                              \
1441   vdup.u32 dx8, rg_dx8[1];                                                     \
1442                                                                                \
1443   vadd.u32 g_block, g_block, dx8;                                              \
1444   vdup.u32 dx8, b_dx8;                                                         \
1445                                                                                \
1446   vadd.u32 b_block, b_block, dx8;                                              \
1447   vmovn.u16 u_whole_8, u_whole;                                                \
1448                                                                                \
1449   vmovn.u16 v_whole_8, v_whole;                                                \
1450                                                                                \
1451   vmovn.u16 b_whole_8, b_whole;                                                \
1452   pld [ fb_ptr ];                                                              \
1453   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1454                                                                                \
1455   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1456   setup_blocks_texture_##swizzling();                                          \
1457                                                                                \
1458   vmovn.u16 r_whole_8, r_whole;                                                \
1459   beq 5f;                                                                      \
1460                                                                                \
1461  4:                                                                            \
1462   vmovn.u16 g_whole_8, g_whole;                                                \
1463   vshrn.u32 u_whole_low, u_block, #16;                                         \
1464                                                                                \
1465   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1466   vshrn.u32 v_whole_low, v_block, #16;                                         \
1467                                                                                \
1468   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1469   vshrn.u32 r_whole_low, r_block, #16;                                         \
1470                                                                                \
1471   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1472   vshrn.u32 g_whole_low, g_block, #16;                                         \
1473                                                                                \
1474   vdup.u32 dx4, uv_dx4[0];                                                     \
1475   vshrn.u32 b_whole_low, b_block, #16;                                         \
1476                                                                                \
1477   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1478   vdup.u32 dx4, uv_dx4[1];                                                     \
1479                                                                                \
1480   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1481   vdup.u32 dx4, rg_dx4[0];                                                     \
1482                                                                                \
1483   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1484   vdup.u32 dx4, rg_dx4[1];                                                     \
1485                                                                                \
1486   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1487   vdup.u32 dx4, b_dx4;                                                         \
1488                                                                                \
1489   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1490   vdup.u32 dx8, uv_dx8[0];                                                     \
1491                                                                                \
1492   vadd.u32 u_block, u_block, dx8;                                              \
1493   vdup.u32 dx8, uv_dx8[1];                                                     \
1494                                                                                \
1495   vadd.u32 v_block, v_block, dx8;                                              \
1496   vdup.u32 dx8, rg_dx8[0];                                                     \
1497                                                                                \
1498   vadd.u32 r_block, r_block, dx8;                                              \
1499   vdup.u32 dx8, rg_dx8[1];                                                     \
1500                                                                                \
1501   vadd.u32 g_block, g_block, dx8;                                              \
1502   vdup.u32 dx8, b_dx8;                                                         \
1503                                                                                \
1504   vadd.u32 b_block, b_block, dx8;                                              \
1505   vmovn.u16 u_whole_8, u_whole;                                                \
1506                                                                                \
1507   add fb_ptr, fb_ptr, #16;                                                     \
1508   vmovn.u16 v_whole_8, v_whole;                                                \
1509                                                                                \
1510   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1511   vmovn.u16 b_whole_8, b_whole;                                                \
1512                                                                                \
1513   pld [ fb_ptr ];                                                              \
1514                                                                                \
1515   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1516   subs span_num_blocks, span_num_blocks, #1;                                   \
1517                                                                                \
1518   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1519   setup_blocks_texture_##swizzling();                                          \
1520                                                                                \
1521   vmovn.u16 r_whole_8, r_whole;                                                \
1522   bne 4b;                                                                      \
1523                                                                                \
1524  5:                                                                            \
1525   vmovn.u16 g_whole_8, g_whole;                                                \
1526   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1527                                                                                \
1528   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1529   vdup.u8 draw_mask, right_mask;                                               \
1530                                                                                \
1531   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1532   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1533   vzip.u8 u_whole_8, v_whole_8;                                                \
1534                                                                                \
1535   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1536   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1537   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1538   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1539   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1540                                                                                \
1541  1:                                                                            \
1542   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1543   add span_b_offset, span_b_offset, #4;                                        \
1544                                                                                \
1545   add span_edge_data, span_edge_data, #8;                                      \
1546   subs num_spans, num_spans, #1;                                               \
1547                                                                                \
1548   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1549   bne 0b;                                                                      \
1550                                                                                \
1551   ldmia sp!, { r4 - r11, pc };                                                 \
1552                                                                                \
1553  2:                                                                            \
1554   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1555   vpush { texture_mask };                                                      \
1556   vpush { uvrg_dx4 };                                                          \
1557                                                                                \
1558   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1559   bl flush_render_block_buffer;                                                \
1560   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1561                                                                                \
1562   vpop { uvrg_dx4 };                                                           \
1563   vpop { texture_mask };                                                       \
1564                                                                                \
1565   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1566   vmov.u8 fb_mask_ptrs, #0;                                                    \
1567                                                                                \
1568   mov num_blocks, span_num_blocks;                                             \
1569   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1570   bal 3b                                                                       \
1571
1572
1573 setup_blocks_shaded_textured_builder(swizzled)
1574 setup_blocks_shaded_textured_builder(unswizzled)
1575
1576
1577 #define setup_blocks_unshaded_textured_builder(swizzling)                      \
1578 .align 3;                                                                      \
1579                                                                                \
1580 function(setup_blocks_unshaded_textured_dithered_##swizzling##_indirect)       \
1581   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1582   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1583                                                                                \
1584   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1585   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1586                                                                                \
1587   cmp num_spans, #0;                                                           \
1588   bxeq lr;                                                                     \
1589                                                                                \
1590   stmdb sp!, { r4 - r11, r14 };                                                \
1591   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1592                                                                                \
1593   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1594                                                                                \
1595   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1596   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1597                                                                                \
1598   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1599   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1600                                                                                \
1601   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1602                                                                                \
1603   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1604                                                                                \
1605  0:                                                                            \
1606   vmov.u8 fb_mask_ptrs, #0;                                                    \
1607                                                                                \
1608   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1609   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1610                                                                                \
1611   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1612   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_out_ptr_offset ];                       \
1613                                                                                \
1614   cmp span_num_blocks, #0;                                                     \
1615   beq 1f;                                                                      \
1616                                                                                \
1617   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1618   add num_blocks, span_num_blocks, num_blocks;                                 \
1619                                                                                \
1620   cmp num_blocks, #MAX_BLOCKS;                                                 \
1621   bgt 2f;                                                                      \
1622                                                                                \
1623  3:                                                                            \
1624   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1625                                                                                \
1626   vdup.u32 v_left_x, left_x;                                                   \
1627   and y, y, #0x3;                                                              \
1628                                                                                \
1629   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1630   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1631                                                                                \
1632   and dither_shift, left_x, #0x03;                                             \
1633                                                                                \
1634   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1635   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1636                                                                                \
1637   mov dither_shift, dither_shift, lsl #3;                                      \
1638   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1639                                                                                \
1640   mov c_32, #32;                                                               \
1641   subs span_num_blocks, span_num_blocks, #1;                                   \
1642                                                                                \
1643   mov dither_row, dither_row, ror dither_shift;                                \
1644                                                                                \
1645   vdup.u32 dither_offsets_short, dither_row;                                   \
1646   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1647                                                                                \
1648   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1649                                                                                \
1650   vdup.u32 u_block, uv[0];                                                     \
1651                                                                                \
1652   vdup.u32 v_block, uv[1];                                                     \
1653   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1654                                                                                \
1655   vadd.u32 u_block, u_block, block_span;                                       \
1656   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1657                                                                                \
1658   vadd.u32 v_block, v_block, block_span;                                       \
1659   add block_ptr_b, block_ptr_a, #16;                                           \
1660                                                                                \
1661   vshrn.u32 u_whole_low, u_block, #16;                                         \
1662   vshrn.u32 v_whole_low, v_block, #16;                                         \
1663                                                                                \
1664   vdup.u32 dx4, uv_dx4[0];                                                     \
1665                                                                                \
1666   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1667   vdup.u32 dx4, uv_dx4[1];                                                     \
1668                                                                                \
1669   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1670   vdup.u32 dx8, uv_dx8[0];                                                     \
1671                                                                                \
1672   vadd.u32 u_block, u_block, dx8;                                              \
1673   vdup.u32 dx8, uv_dx8[1];                                                     \
1674                                                                                \
1675   vadd.u32 v_block, v_block, dx8;                                              \
1676   vmovn.u16 u_whole_8, u_whole;                                                \
1677                                                                                \
1678   vmovn.u16 v_whole_8, v_whole;                                                \
1679                                                                                \
1680   pld [ fb_ptr ];                                                              \
1681   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1682                                                                                \
1683   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1684   setup_blocks_texture_##swizzling();                                          \
1685                                                                                \
1686   beq 5f;                                                                      \
1687                                                                                \
1688  4:                                                                            \
1689   vshrn.u32 u_whole_low, u_block, #16;                                         \
1690                                                                                \
1691   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1692   vshrn.u32 v_whole_low, v_block, #16;                                         \
1693                                                                                \
1694   add block_ptr_b, block_ptr_b, #32;                                           \
1695   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1696                                                                                \
1697   vdup.u32 dx4, uv_dx4[0];                                                     \
1698   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1699   vdup.u32 dx4, uv_dx4[1];                                                     \
1700                                                                                \
1701   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1702   vdup.u32 dx8, uv_dx8[0];                                                     \
1703                                                                                \
1704   vadd.u32 u_block, u_block, dx8;                                              \
1705   vdup.u32 dx8, uv_dx8[1];                                                     \
1706                                                                                \
1707   vadd.u32 v_block, v_block, dx8;                                              \
1708   vmovn.u16 u_whole_8, u_whole;                                                \
1709                                                                                \
1710   add fb_ptr, fb_ptr, #16;                                                     \
1711   vmovn.u16 v_whole_8, v_whole;                                                \
1712                                                                                \
1713   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1714   pld [ fb_ptr ];                                                              \
1715                                                                                \
1716   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1717   subs span_num_blocks, span_num_blocks, #1;                                   \
1718                                                                                \
1719   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1720   setup_blocks_texture_##swizzling();                                          \
1721                                                                                \
1722   bne 4b;                                                                      \
1723                                                                                \
1724  5:                                                                            \
1725   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1726                                                                                \
1727   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1728   vdup.u8 draw_mask, right_mask;                                               \
1729                                                                                \
1730   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1731   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1732   vzip.u8 u_whole_8, v_whole_8;                                                \
1733                                                                                \
1734   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1735   add block_ptr_b, block_ptr_b, #32;                                           \
1736   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1737   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1738   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1739                                                                                \
1740  1:                                                                            \
1741   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1742   add span_edge_data, span_edge_data, #8;                                      \
1743   subs num_spans, num_spans, #1;                                               \
1744                                                                                \
1745   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1746   bne 0b;                                                                      \
1747                                                                                \
1748   ldmia sp!, { r4 - r11, pc };                                                 \
1749                                                                                \
1750  2:                                                                            \
1751   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1752   vpush { texture_mask };                                                      \
1753   vpush { uvrg_dx4 };                                                          \
1754                                                                                \
1755   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1756   bl flush_render_block_buffer;                                                \
1757   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1758                                                                                \
1759   vpop { uvrg_dx4 };                                                           \
1760   vpop { texture_mask };                                                       \
1761                                                                                \
1762   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1763   vmov.u8 fb_mask_ptrs, #0;                                                    \
1764                                                                                \
1765   mov num_blocks, span_num_blocks;                                             \
1766   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1767   bal 3b                                                                       \
1768
1769
1770 setup_blocks_unshaded_textured_builder(swizzled)
1771 setup_blocks_unshaded_textured_builder(unswizzled)
1772
1773
1774 .align 3
1775
1776 function(setup_blocks_unshaded_untextured_undithered_unswizzled_indirect)
1777   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1778   veor.u32 draw_mask, draw_mask, draw_mask
1779
1780   cmp num_spans, #0
1781   bxeq lr
1782
1783   stmdb sp!, { r4 - r11, r14 }
1784   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1785
1786   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1787
1788   ubfx color_r, color, #3, #5
1789   ubfx color_g, color, #11, #5
1790   ubfx color_b, color, #19, #5
1791
1792   orr color, color_r, color_b, lsl #10
1793   orr color, color, color_g, lsl #5
1794
1795   vdup.u16 colors, color
1796
1797   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1798   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1799
1800   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1801   add block_ptr_a, block_ptr_a, num_blocks, lsl #6
1802
1803  0:
1804   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1805   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1806
1807   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_out_ptr_offset ]
1808
1809   cmp span_num_blocks, #0
1810   beq 1f
1811
1812   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1813   add num_blocks, span_num_blocks, num_blocks
1814
1815   cmp num_blocks, #MAX_BLOCKS
1816   bgt 2f
1817
1818  3:
1819   add fb_ptr, fb_ptr, y, lsl #11
1820   and y, y, #0x3
1821
1822   add fb_ptr, fb_ptr, left_x, lsl #1
1823   mov c_32, #32
1824
1825   subs span_num_blocks, span_num_blocks, #1
1826
1827   add block_ptr_b, block_ptr_a, #16
1828   pld [ fb_ptr ]
1829
1830   vmov.u32 fb_mask_ptrs[1], fb_ptr
1831   beq 5f
1832
1833  4:
1834   vst1.u32 { draw_mask }, [ block_ptr_a, :128 ], c_32
1835   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1836   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1837
1838   add fb_ptr, fb_ptr, #16
1839   add block_ptr_b, block_ptr_b, #32
1840
1841   pld [ fb_ptr ]
1842
1843   vmov.u32 fb_mask_ptrs[1], fb_ptr
1844   subs span_num_blocks, span_num_blocks, #1
1845
1846   bne 4b
1847
1848  5:
1849   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1850
1851   vdup.u8 draw_mask_edge, right_mask
1852   vtst.u16 draw_mask_edge, draw_mask_edge, test_mask
1853
1854   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1855   vst1.u32 { draw_mask_edge }, [ block_ptr_a, :128 ], c_32
1856   add block_ptr_b, block_ptr_b, #32
1857   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1858
1859  1:
1860   add span_edge_data, span_edge_data, #8
1861   subs num_spans, num_spans, #1
1862
1863   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1864   bne 0b
1865
1866   ldmia sp!, { r4 - r11, pc }
1867                                                                            
1868  2:
1869   vpush { colors }
1870
1871   stmdb sp!, { r0 - r3, r12, r14 }
1872   bl flush_render_block_buffer
1873   ldmia sp!, { r0 - r3, r12, r14 }
1874
1875   vpop { colors }
1876
1877   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1878   veor.u32 draw_mask, draw_mask, draw_mask
1879
1880   mov num_blocks, span_num_blocks
1881   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1882   bal 3b
1883
1884
1885 #define mask_msb_scalar                                   r14
1886
1887 #define msb_mask                                          q15
1888
1889 #define pixels_low                                        d16
1890
1891 #define msb_mask_low                                      d30
1892 #define msb_mask_high                                     d31
1893
1894
1895 .align 3
1896
1897 function(setup_blocks_unshaded_untextured_undithered_unswizzled_direct)
1898   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1899
1900   cmp num_spans, #0
1901   bxeq lr
1902
1903   stmdb sp!, { r4 - r11, r14 }
1904
1905   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1906
1907   ubfx color_r, color, #3, #5
1908   ubfx color_g, color, #11, #5
1909
1910   ldrh mask_msb_scalar, [ psx_gpu, #psx_gpu_mask_msb_offset ]
1911   ubfx color_b, color, #19, #5
1912
1913   orr color, color_r, color_b, lsl #10
1914   orr color, color, color_g, lsl #5
1915   orr color, color, mask_msb_scalar
1916
1917   vdup.u16 colors, color
1918
1919   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1920   orr color, color, lsl #16
1921
1922
1923  0:
1924   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1925   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1926
1927   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_out_ptr_offset ]
1928
1929   cmp span_num_blocks, #0
1930   beq 1f
1931
1932   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1933
1934   add fb_ptr, fb_ptr, y, lsl #11
1935   subs span_num_blocks, span_num_blocks, #1
1936
1937   add fb_ptr, fb_ptr, left_x, lsl #1
1938   beq 3f
1939
1940  2:
1941   vst1.u32 { colors }, [ fb_ptr ]!
1942   subs span_num_blocks, span_num_blocks, #1
1943
1944   bne 2b
1945
1946  3:
1947   ldrb right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1948
1949   cmp right_mask, #0x0
1950   beq 5f
1951
1952   tst right_mask, #0xF
1953   streq color, [ fb_ptr ], #4
1954   moveq right_mask, right_mask, lsr #4
1955   streq color, [ fb_ptr ], #4
1956
1957   tst right_mask, #0x3
1958   streq color, [ fb_ptr ], #4
1959   moveq right_mask, right_mask, lsr #2
1960
1961   tst right_mask, #0x1
1962   streqh color, [ fb_ptr ]
1963
1964  1:
1965   add span_edge_data, span_edge_data, #8
1966   subs num_spans, num_spans, #1
1967   bne 0b
1968
1969   ldmia sp!, { r4 - r11, pc }
1970                                                                            
1971  5:
1972   vst1.u32 { colors }, [ fb_ptr ]
1973   bal 1b
1974
1975
1976 #undef c_64
1977
1978 #define c_64                                              r7
1979 #define rg_dx_ptr                                         r2
1980
1981
1982 #undef r_block
1983 #undef g_block
1984 #undef b_block
1985 #undef r_whole
1986 #undef g_whole
1987 #undef b_whole
1988 #undef r_whole_low
1989 #undef r_whole_high
1990 #undef g_whole_low
1991 #undef g_whole_high
1992 #undef b_whole_low
1993 #undef b_whole_high
1994 #undef r_whole_8
1995 #undef g_whole_8
1996 #undef b_whole_8
1997 #undef dither_offsets
1998 #undef rg_dx4
1999 #undef rg_dx8
2000 #undef dx4
2001 #undef dx8
2002 #undef v_left_x
2003 #undef uvrg
2004 #undef block_span
2005 #undef rg
2006 #undef draw_mask
2007 #undef test_mask
2008
2009 #define r_block                                           q0
2010 #define g_block                                           q1
2011 #define b_block                                           q2
2012
2013 #define r_whole                                           q3
2014 #define g_whole                                           q4
2015 #define b_whole                                           q5
2016
2017 #define r_whole_low                                       d6
2018 #define r_whole_high                                      d7
2019 #define g_whole_low                                       d8
2020 #define g_whole_high                                      d9
2021 #define b_whole_low                                       d10
2022 #define b_whole_high                                      d11
2023
2024 #define gb_whole_8                                        q6
2025
2026 #define g_whole_8                                         d12
2027 #define b_whole_8                                         d13
2028
2029 #define r_whole_8                                         d14
2030
2031 #define pixels                                            q8
2032
2033 #define rg_dx4                                            d18
2034 #define rg_dx8                                            d19
2035
2036 #define dx4                                               q10
2037 #define dx8                                               q10
2038
2039 #define v_left_x                                          d6
2040 #define uvrg                                              q4
2041 #define block_span                                        q5
2042
2043 #define rg                                                d9
2044
2045 #define d64_1                                             d22
2046 #define d64_128                                           d23
2047
2048 #define d128_4                                            q12
2049 #define d128_0x7                                          q13
2050
2051 #define d64_4                                             d24
2052
2053 #define dither_offsets                                    q14
2054 #define draw_mask                                         q15
2055
2056 #define dither_offsets_low                                d28
2057
2058 #define rg_dx                                             d0
2059 #define test_mask                                         q10
2060
2061
2062 #define setup_blocks_shaded_untextured_dither_a_dithered()                     \
2063   vqadd.u8 r_whole_8, r_whole_8, dither_offsets_low;                           \
2064   vqadd.u8 gb_whole_8, gb_whole_8, dither_offsets;                             \
2065
2066 #define setup_blocks_shaded_untextured_dither_b_dithered()                     \
2067   vqsub.u8 r_whole_8, r_whole_8, d64_4;                                        \
2068   vqsub.u8 gb_whole_8, gb_whole_8, d128_4                                      \
2069
2070 #define setup_blocks_shaded_untextured_dither_a_undithered()                   \
2071
2072 #define setup_blocks_shaded_untextured_dither_b_undithered()                   \
2073
2074
2075 #define setup_blocks_shaded_untextured_indirect_builder(dithering)             \
2076 .align 3;                                                                      \
2077                                                                                \
2078 function(setup_blocks_shaded_untextured_##dithering##_unswizzled_indirect)     \
2079   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
2080   add rg_dx_ptr, psx_gpu, #(psx_gpu_uvrg_dx_offset + 8);                       \
2081                                                                                \
2082   vld1.u32 { rg_dx }, [ rg_dx_ptr, :64 ];                                      \
2083                                                                                \
2084   cmp num_spans, #0;                                                           \
2085   bxeq lr;                                                                     \
2086                                                                                \
2087   stmdb sp!, { r4 - r11, r14 };                                                \
2088   vshl.u32 rg_dx4, rg_dx, #2;                                                  \
2089                                                                                \
2090   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
2091   vshl.u32 rg_dx8, rg_dx, #3;                                                  \
2092                                                                                \
2093   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
2094                                                                                \
2095   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
2096   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
2097                                                                                \
2098   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
2099   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
2100                                                                                \
2101   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
2102   vmov.u8 d64_1, #1;                                                           \
2103                                                                                \
2104   vmov.u8 d128_4, #4;                                                          \
2105   vmov.u8 d64_128, #128;                                                       \
2106                                                                                \
2107   vmov.u8 d128_0x7, #0x7;                                                      \
2108                                                                                \
2109  0:                                                                            \
2110   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
2111   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
2112                                                                                \
2113   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
2114   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_out_ptr_offset ];                       \
2115                                                                                \
2116   cmp span_num_blocks, #0;                                                     \
2117   beq 1f;                                                                      \
2118                                                                                \
2119   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
2120   add num_blocks, span_num_blocks, num_blocks;                                 \
2121                                                                                \
2122   cmp num_blocks, #MAX_BLOCKS;                                                 \
2123   bgt 2f;                                                                      \
2124                                                                                \
2125  3:                                                                            \
2126   ldr b, [ span_b_offset ];                                                    \
2127   add fb_ptr, fb_ptr, y, lsl #11;                                              \
2128                                                                                \
2129   vdup.u32 v_left_x, left_x;                                                   \
2130   and y, y, #0x3;                                                              \
2131                                                                                \
2132   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
2133   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
2134                                                                                \
2135   mla b, b_dx, left_x, b;                                                      \
2136   and dither_shift, left_x, #0x03;                                             \
2137                                                                                \
2138   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
2139   vshr.u32 rg_dx, rg_dx4, #2;                                                  \
2140                                                                                \
2141   mov dither_shift, dither_shift, lsl #3;                                      \
2142   vmla.u32 rg, rg_dx, v_left_x;                                                \
2143                                                                                \
2144   mov c_64, #64;                                                               \
2145   subs span_num_blocks, span_num_blocks, #1;                                   \
2146                                                                                \
2147   mov dither_row, dither_row, ror dither_shift;                                \
2148   mov b_dx4, b_dx, lsl #2;                                                     \
2149                                                                                \
2150   vdup.u32 dither_offsets, dither_row;                                         \
2151   add block_span_ptr, psx_gpu, #psx_gpu_r_block_span_offset;                   \
2152                                                                                \
2153   vdup.u32 b_block, b;                                                         \
2154   vadd.u8 dither_offsets, dither_offsets, d128_4;                              \
2155                                                                                \
2156   mov b_dx8, b_dx, lsl #3;                                                     \
2157   vdup.u32 r_block, rg[0];                                                     \
2158   vdup.u32 g_block, rg[1];                                                     \
2159                                                                                \
2160   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2161                                                                                \
2162   vadd.u32 r_block, r_block, block_span;                                       \
2163   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2164                                                                                \
2165   vadd.u32 g_block, g_block, block_span;                                       \
2166   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
2167                                                                                \
2168   vadd.u32 b_block, b_block, block_span;                                       \
2169   add block_ptr_b, block_ptr_a, #16;                                           \
2170                                                                                \
2171   vshrn.u32 r_whole_low, r_block, #16;                                         \
2172   vshrn.u32 g_whole_low, g_block, #16;                                         \
2173   vshrn.u32 b_whole_low, b_block, #16;                                         \
2174   vdup.u32 dx4, rg_dx4[0];                                                     \
2175                                                                                \
2176   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2177   vdup.u32 dx4, rg_dx4[1];                                                     \
2178                                                                                \
2179   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2180   vdup.u32 dx4, b_dx4;                                                         \
2181                                                                                \
2182   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2183   vdup.u32 dx8, rg_dx8[0];                                                     \
2184                                                                                \
2185   vadd.u32 r_block, r_block, dx8;                                              \
2186   vdup.u32 dx8, rg_dx8[1];                                                     \
2187                                                                                \
2188   vadd.u32 g_block, g_block, dx8;                                              \
2189   vdup.u32 dx8, b_dx8;                                                         \
2190                                                                                \
2191   vadd.u32 b_block, b_block, dx8;                                              \
2192                                                                                \
2193   vmovn.u16 r_whole_8, r_whole;                                                \
2194   vmovn.u16 g_whole_8, g_whole;                                                \
2195   vmovn.u16 b_whole_8, b_whole;                                                \
2196                                                                                \
2197   beq 5f;                                                                      \
2198   veor.u32 draw_mask, draw_mask, draw_mask;                                    \
2199                                                                                \
2200  4:                                                                            \
2201   setup_blocks_shaded_untextured_dither_a_##dithering();                       \
2202   vshrn.u32 r_whole_low, r_block, #16;                                         \
2203                                                                                \
2204   setup_blocks_shaded_untextured_dither_b_##dithering();                       \
2205   vshrn.u32 g_whole_low, g_block, #16;                                         \
2206                                                                                \
2207   vshrn.u32 b_whole_low, b_block, #16;                                         \
2208   str fb_ptr, [ block_ptr_a, #44 ];                                            \
2209                                                                                \
2210   vdup.u32 dx4, rg_dx4[0];                                                     \
2211   vshr.u8 r_whole_8, r_whole_8, #3;                                            \
2212   vbic.u8 gb_whole_8, gb_whole_8, d128_0x7;                                    \
2213                                                                                \
2214   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2215   vdup.u32 dx4, rg_dx4[1];                                                     \
2216                                                                                \
2217   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2218   vdup.u32 dx4, b_dx4;                                                         \
2219                                                                                \
2220   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2221   vdup.u32 dx8, rg_dx8[0];                                                     \
2222                                                                                \
2223   vmull.u8 pixels, r_whole_8, d64_1;                                           \
2224   vmlal.u8 pixels, g_whole_8, d64_4;                                           \
2225   vmlal.u8 pixels, b_whole_8, d64_128;                                         \
2226                                                                                \
2227   vadd.u32 r_block, r_block, dx8;                                              \