381f3a9ead3319978b7b7829c3ad218a5afc7aeb
[pcsx_rearmed.git] / plugins / gpu_neon / psx_gpu / psx_gpu_arm_neon.S
1 /*
2  * Copyright (C) 2011 Gilead Kutnick "Exophase" <exophase@gmail.com>
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of
7  * the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  */
14
15 #define MAX_SPANS                                         512
16 #define MAX_BLOCKS                                        64
17 #define MAX_BLOCKS_PER_ROW                                128
18
19 #define psx_gpu_test_mask_offset                          0
20 #define psx_gpu_uvrg_offset                               16
21 #define psx_gpu_uvrg_dx_offset                            32
22 #define psx_gpu_uvrg_dy_offset                            48
23 #define psx_gpu_u_block_span_offset                       64
24 #define psx_gpu_v_block_span_offset                       80
25 #define psx_gpu_r_block_span_offset                       96
26 #define psx_gpu_g_block_span_offset                       112
27 #define psx_gpu_b_block_span_offset                       128
28
29 #define psx_gpu_b_dx_offset                               132
30
31 #define psx_gpu_b_offset                                  144
32 #define psx_gpu_b_dy_offset                               148
33 #define psx_gpu_triangle_area_offset                      152
34 #define psx_gpu_texture_window_settings_offset            156
35 #define psx_gpu_current_texture_mask_offset               160
36 #define psx_gpu_viewport_mask_offset                      164
37 #define psx_gpu_dirty_textures_4bpp_mask_offset           168
38 #define psx_gpu_dirty_textures_8bpp_mask_offset           172
39 #define psx_gpu_dirty_textures_8bpp_alternate_mask_offset 176
40 #define psx_gpu_triangle_color_offset                     180
41 #define psx_gpu_primitive_color_offset                    184
42 #define psx_gpu_dither_table_offset                       188
43 #define psx_gpu_render_block_handler_offset               204
44 #define psx_gpu_texture_page_ptr_offset                   208
45 #define psx_gpu_clut_ptr_offset                           212
46 #define psx_gpu_vram_ptr_offset                           216
47
48 #define psx_gpu_render_state_base_offset                  220
49 #define psx_gpu_render_state_offset                       222
50 #define psx_gpu_num_spans_offset                          224
51 #define psx_gpu_num_blocks_offset                         226
52 #define psx_gpu_offset_x_offset                           228
53 #define psx_gpu_offset_y_offset                           230
54 #define psx_gpu_clut_settings_offset                      232
55 #define psx_gpu_texture_settings_offset                   234
56 #define psx_gpu_viewport_start_x_offset                   236
57 #define psx_gpu_viewport_start_y_offset                   238
58 #define psx_gpu_viewport_end_x_offset                     240
59 #define psx_gpu_viewport_end_y_offset                     242
60 #define psx_gpu_mask_msb_offset                           244
61                                                           
62 #define psx_gpu_triangle_winding_offset                   246
63 #define psx_gpu_display_area_draw_enable_offset           247
64 #define psx_gpu_current_texture_page_offset               248
65 #define psx_gpu_last_8bpp_texture_page_offset             249
66 #define psx_gpu_texture_mask_width_offset                 250
67 #define psx_gpu_texture_mask_height_offset                251
68 #define psx_gpu_texture_window_x_offset                   252
69 #define psx_gpu_texture_window_y_offset                   253
70 #define psx_gpu_primitive_type_offset                     254
71
72 #define psx_gpu_reserved_a_offset                         255
73
74 #define psx_gpu_blocks_offset                             0x0100
75 #define psx_gpu_span_uvrg_offset_offset                   0x2100
76 #define psx_gpu_span_edge_data_offset                     0x4100
77 #define psx_gpu_span_b_offset_offset                      0x5100
78
79 #define psx_gpu__vram_offset                              0x005900
80
81 #define edge_data_left_x_offset                           0
82 #define edge_data_num_blocks_offset                       2
83 #define edge_data_right_mask_offset                       4
84 #define edge_data_y_offset                                6
85
86
87 #define psx_gpu                                           r0
88 #define v_a                                               r1
89 #define v_b                                               r2
90 #define v_c                                               r3
91
92 #define x0                                                r4
93 #define x1                                                r5
94 #define x2                                                r6
95 #define x0_x1                                             r5
96 #define x1_x2                                             r6
97 #define y0                                                r7
98 #define y1                                                r8
99 #define y2                                                r9
100 #define y0_y1                                             r7
101 #define y1_y2                                             r8
102 #define b0                                                r9
103 #define b1                                                r10
104 #define b2                                                r11
105 #define b0_b1                                             r10
106 #define b1_b2                                             r11
107
108
109 #define area_r_s                                          r5
110
111 #define g_bx0                                             r2
112 #define g_bx                                              r3
113 #define g_bx2                                             r4
114 #define g_bx3                                             r5
115 #define b_base                                            r6
116 #define g_by                                              r8
117
118 #define gs_bx                                             r7
119 #define gs_by                                             r10
120
121 #define ga_bx                                             g_bx
122 #define ga_by                                             g_by
123
124 #define gw_bx_h                                           g_bx
125 #define gw_by_h                                           g_by
126
127 #define gw_bx_l                                           r11
128 #define gw_by_l                                           gw_bx_l
129
130 #define store_a                                           r0
131 #define store_b                                           r1
132 #define store_inc                                         r5
133
134
135 #define v0                                                q0
136 #define uvrgb0                                            d0
137 #define x0_y0                                             d1
138
139 #define v1                                                q1
140 #define uvrgb1                                            d2
141 #define x1_y1                                             d3
142
143 #define v2                                                q2
144 #define uvrgb2                                            d4
145 #define x2_y2                                             d5
146
147 #define x0_ab                                             q3
148 #define uvrg_xxxx0                                        q3
149 #define uvrg0                                             d6
150 #define xxxx0                                             d7
151
152 #define x1_ab                                             q4
153 #define uvrg_xxxx1                                        q4
154 #define uvrg1                                             d8
155 #define xxxx1                                             d9
156
157 #define x2_ab                                             q5
158 #define uvrg_xxxx2                                        q5
159 #define uvrg2                                             d10
160 #define xxxx2                                             d11
161
162 #define y0_ab                                             q6
163 #define yyyy_uvrg0                                        q6
164 #define yyyy0                                             d12
165 #define uvrg0b                                            d13
166
167 #define y1_ab                                             q7
168 #define yyyy_uvrg1                                        q7
169 #define yyyy1                                             d14
170 #define uvrg1b                                            d15
171
172 #define y2_ab                                             q8
173 #define yyyy_uvrg2                                        q8
174 #define yyyy2                                             d16
175 #define uvrg2b                                            d17
176
177 #define d0_ab                                             q9
178 #define d0_a                                              d18
179 #define d0_b                                              d19
180
181 #define d1_ab                                             q10
182 #define d1_a                                              d20
183 #define d1_b                                              d21
184
185 #define d2_ab                                             q11
186 #define d2_a                                              d22
187 #define d2_b                                              d23
188
189 #define d3_ab                                             q12
190 #define d3_a                                              d24
191 #define d3_b                                              d25
192
193 #define ga_uvrg_x                                         q1
194 #define ga_uvrg_y                                         q4
195
196 #define dx                                                x0_x1
197 #define dy                                                y0_y1
198 #define db                                                b0_b1
199
200 #define uvrg_base                                         q11
201
202 #define gs_uvrg_x                                         q5
203 #define gs_uvrg_y                                         q6
204
205 #define g_uvrg_x                                          q1
206 #define ga_uv_x                                           d2
207 #define g_uv_x                                            d2
208 #define ga_rg_x                                           d3
209 #define g_rg_x                                            d3
210
211 #define g_uvrg_y                                          q4
212 #define ga_uv_y                                           d8
213 #define g_uv_y                                            d8
214 #define ga_rg_y                                           d9
215 #define g_rg_y                                            d9
216
217 #define gw_uv_x                                           q1
218 #define gw_rg_x                                           q2
219 #define gw_uv_y                                           q4
220 #define gw_rg_y                                           q3
221
222 #define w_mask                                            q9
223 #define w_mask_l                                          d18
224
225 #define r_shift                                           q10
226
227 #define uvrg_dx0                                          q0
228 #define uvrg_dx0l                                         d0
229 #define uvrg_dx0h                                         d1
230
231 #define uvrg_dx1                                          q1
232 #define uvrg_dx1l                                         d2
233 #define uvrg_dx1h                                         d3
234
235 #define uvrg_dx2                                          q2
236 #define uvrg_dx2l                                         d4
237 #define uvrg_dx2h                                         d5
238
239 #define uvrg_dx3                                          q3
240 #define uvrg_dx3l                                         d6
241 #define uvrg_dx3h                                         d7
242
243
244 .align 4
245
246 #define function(name)                                                         \
247   .global name;                                                                \
248   name:                                                                        \
249
250 @ r0: psx_gpu
251 @ r1: v_a
252 @ r2: v_b
253 @ r3: v_c
254
255 function(compute_all_gradients)
256   // First compute the triangle area reciprocal and shift. The division will
257   // happen concurrently with much of the work which follows.
258   @ r12 = psx_gpu->triangle_area
259   ldr r12, [ psx_gpu, #psx_gpu_triangle_area_offset ]
260   stmdb sp!, { r4 - r11, lr }
261
262   @ load exponent of 62 into upper half of double
263   movw r4, #0
264   clz r14, r12                       @ r14 = shift
265
266   movt r4, #((62 + 1023) << 4)
267   mov r12, r12, lsl r14              @ r12 = triangle_area_normalized
268
269   @ load area normalized into lower half of double
270   mov r5, r12, lsr #10
271   vmov.f64 d30, r5, r4               @ d30 = (1 << 62) + ta_n
272
273   movt r4, #((1022 + 31) << 4)
274   mov r5, r12, lsl #20
275
276   add r4, r4, r12, lsr #11
277   vmov.f64 d31, r5, r4
278
279   vdiv.f64 d30, d30, d31             @ d30 = ((1 << 62) + ta_n) / ta_n
280
281   // ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0)) =
282   // ( d0       *  d1      ) - ( d2       *  d3      ) =
283   // ( m0                  ) - ( m1                  ) = gradient
284
285   // This is split to do 12 elements at a time over three sets: a, b, and c.
286   // Technically we only need to do 10 elements (uvrgb_x and uvrgb_y), so
287   // two of the slots are unused.
288
289   // Inputs are all 16-bit signed. The m0/m1 results are 32-bit signed, as
290   // is g.
291
292   // First type is:  uvrg bxxx xxxx 
293   // Second type is: yyyy ybyy uvrg 
294   // Since x_a and y_c are the same the same variable is used for both. 
295
296   vld1.u32 { v0 }, [ v_a, : 128 ]    @ v0 = { uvrg0, b0, x0, y0 }
297   ldrsh x0, [ v_a, #8 ]              @ load x0
298
299   vld1.u32 { v1 }, [ v_b, : 128 ]    @ v1 = { uvrg1, b1, x1, y1}
300   ldrh x1, [ v_b, #8 ]               @ load x1
301
302   vld1.u32 { v2 }, [ v_c, : 128 ]    @ v2 = { uvrg2, b2, x2, y2 }
303   ldrh x2, [ v_c, #8 ]               @ load x2
304
305   vmovl.u8 uvrg_xxxx0, uvrgb0        @ uvrg_xxxx0 = { uv0, rg0, b0-, -- }
306   ldrh y0, [ v_a, #10 ]              @ load y0
307
308   vmovl.u8 uvrg_xxxx1, uvrgb1        @ uvrg_xxxx1 = { uv1, rg1, b1-, -- }
309   ldrh y1, [ v_b, #10 ]              @ load y1
310
311   vmovl.u8 uvrg_xxxx2, uvrgb2        @ uvrg_xxxx2 = { uv2, rg2, b2-, -- }
312   ldrh y2, [ v_c, #10 ]              @ load y2
313
314   vmov.u8 uvrg0b, uvrg0              @ uvrg0b = { uv0, rg0 }
315   vdup.u16 xxxx0, x0_y0[0]           @ xxxx0 = { xx0, xx0 }
316
317   orr x1_x2, x1, x2, lsl #16         @ x1_x2 = { x1, x2 }
318   pkhbt x0_x1, x0, x1, lsl #16       @ x0_x1 = { x0, x1 }
319
320   vmov.u8 uvrg1b, uvrg1              @ uvrg1b = { uv1, rg1 }
321   vdup.u16 xxxx1, x1_y1[0]           @ xxxx1 = { xx1, xx1 }
322
323   vmov.u8 uvrg2b, uvrg2              @ uvrg2b = { uv2, rg2 }
324   vdup.u16 xxxx2, x2_y2[0]           @ xxxx2 = { xx2, xx2 }
325
326   ldrb b2, [ v_c, #4 ]               @ load b2
327   orr y0_y1, y0, y1, lsl #16         @ y0_y1 = { y0, y1 }
328
329   ldrb b1, [ v_b, #4 ]               @ load b1
330   orr y1_y2, y1, y2, lsl #16         @ y1_y2 = { y1, y2 }
331
332   vdup.u16 yyyy0, x0_y0[1]           @ yyyy0 = { yy0, yy0 }
333   vsub.s16 d0_ab, x1_ab, x0_ab
334
335   ldrb b0, [ v_a, #4 ]               @ load b0
336   orr b1_b2, b1, b2, lsl #16         @ b1_b2 = { b1, b2 }
337
338   vdup.u16 yyyy1, x1_y1[1]           @ yyyy1 = { yy1, yy1 }
339   vsub.s16 d2_ab, x2_ab, x1_ab
340
341   vdup.u16 yyyy2, x2_y2[1]           @ yyyy2 = { yy2, yy2 }
342   vsub.s16 d1_ab, y2_ab, y1_ab
343
344   orr b0_b1, b0, b1, lsl #16         @ b1_b2 = { b1, b2 }
345   ssub16 dx, x1_x2, x0_x1            @ dx = { x1 - x0, x2 - x1 }
346
347   ssub16 dy, y1_y2, y0_y1            @ dy = { y1 - y0, y2 - y1 }
348   ssub16 db, b1_b2, b0_b1            @ db = { b1 - b0, b2 - b1 }
349
350   vsub.s16 d3_ab, y1_ab, y0_ab
351   smusdx ga_by, dx, db               @ ga_by = ((x1 - x0) * (b2 - b1)) -
352                                      @         ((x2 - X1) * (b1 - b0)) 
353   vmull.s16 ga_uvrg_x, d0_a, d1_a
354   smusdx ga_bx, db, dy               @ ga_bx = ((b1 - b0) * (y2 - y1)) -
355                                      @         ((b2 - b1) * (y1 - y0))
356   vmlsl.s16 ga_uvrg_x, d2_a, d3_a
357   movs gs_bx, ga_bx, asr #31
358
359   vmull.s16 ga_uvrg_y, d0_b, d1_b
360   rsbmi ga_bx, ga_bx, #0
361
362   vmlsl.s16 ga_uvrg_y, d2_b, d3_b
363   movs gs_by, ga_by, asr #31
364
365   vshr.u64 d0, d30, #22
366   mov b_base, b0, lsl #16
367
368   rsbmi ga_by, ga_by, #0
369   vclt.s32 gs_uvrg_x, ga_uvrg_x, #0  @ gs_uvrg_x = ga_uvrg_x < 0
370
371   @ r12 = psx_gpu->triangle_winding_offset
372   ldrb r12, [ psx_gpu, #psx_gpu_triangle_winding_offset ]
373   vclt.s32 gs_uvrg_y, ga_uvrg_y, #0  @ gs_uvrg_y = ga_uvrg_y < 0
374
375   add b_base, b_base, #0x8000
376   rsb r12, r12, #0                   @ r12 = -(triangle->winding)
377
378   vdup.u32 w_mask, r12               @ w_mask = { -w, -w, -w, -w }
379   sub r14, r14, #(62 - 12)           @ r14 = shift - (62 - FIXED_BITS)
380
381   vshll.u16 uvrg_base, uvrg0, #16    @ uvrg_base = uvrg0 << 16
382   vdup.u32 r_shift, r14              @ r_shift = { shift, shift, shift, shift }
383
384   vorr.u32 uvrg_base, #0x8000
385   vabs.s32 ga_uvrg_x, ga_uvrg_x      @ ga_uvrg_x = abs(ga_uvrg_x)
386
387   vmov area_r_s, s0                  @ area_r_s = triangle_reciprocal
388   vabs.s32 ga_uvrg_y, ga_uvrg_y      @ ga_uvrg_y = abs(ga_uvrg_y)
389
390   vmull.u32 gw_rg_x, ga_rg_x, d0[0]
391   vmull.u32 gw_uv_x, ga_uv_x, d0[0]
392   vmull.u32 gw_rg_y, ga_rg_y, d0[0]
393   vmull.u32 gw_uv_y, ga_uv_y, d0[0]
394
395   vshl.u64 gw_rg_x, gw_rg_x, r_shift
396   vshl.u64 gw_uv_x, gw_uv_x, r_shift
397   vshl.u64 gw_rg_y, gw_rg_y, r_shift
398   vshl.u64 gw_uv_y, gw_uv_y, r_shift
399
400   veor.u32 gs_uvrg_x, gs_uvrg_x, w_mask
401   vmovn.u64 g_uv_x, gw_uv_x
402
403   veor.u32 gs_uvrg_y, gs_uvrg_y, w_mask
404   vmovn.u64 g_rg_x, gw_rg_x
405
406   veor.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
407   vmovn.u64 g_uv_y, gw_uv_y
408
409   vsub.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
410   vmovn.u64 g_rg_y, gw_rg_y
411
412   veor.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
413   mov ga_bx, ga_bx, lsl #13
414
415   vsub.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
416   mov ga_by, ga_by, lsl #13
417
418   vdup.u32 x0_y0, x0
419   umull gw_bx_l, gw_bx_h, ga_bx, area_r_s
420
421   vshl.u32 g_uvrg_x, g_uvrg_x, #4
422   vshl.u32 g_uvrg_y, g_uvrg_y, #4
423
424   umull gw_by_l, gw_by_h, ga_by, area_r_s
425   vmls.s32 uvrg_base, ga_uvrg_x, x0_y0[0]
426
427   eor gs_bx, gs_bx, r12
428   vadd.u32 uvrg_dx2, uvrg_dx1, uvrg_dx1
429
430   veor.u32 uvrg_dx0, uvrg_dx0, uvrg_dx0
431   eor gs_by, gs_by, r12
432
433   rsb r11, r14, #0                   @ r11 = negative shift for scalar lsr
434   add store_a, psx_gpu, #psx_gpu_uvrg_offset
435
436   sub r11, r11, #(32 - 13)
437
438   add store_b, store_a, #16
439   mov store_inc, #32
440
441   vadd.u32 uvrg_dx3, uvrg_dx2, uvrg_dx1
442   vst1.u32 { uvrg_base }, [ store_a, : 128 ], store_inc
443
444   vst1.u32 { uvrg_dx1 }, [ store_b, : 128 ], store_inc
445   mov g_bx, gw_bx_h, lsr r11
446
447   vst1.u32 { g_uvrg_y }, [ store_a, : 128 ], store_inc
448   mov g_by, gw_by_h, lsr r11
449
450   vst4.u32 { uvrg_dx0l, uvrg_dx1l, uvrg_dx2l, uvrg_dx3l },                     \
451    [ store_b, : 128 ], store_inc
452   eor g_bx, g_bx, gs_bx
453
454   vst4.u32 { uvrg_dx0h, uvrg_dx1h, uvrg_dx2h, uvrg_dx3h },                     \
455    [ store_b, : 128 ], store_inc
456   sub g_bx, g_bx, gs_bx
457
458   lsl g_bx, g_bx, #4  
459   eor g_by, g_by, gs_by
460
461   mls b_base, g_bx, x0, b_base
462   sub g_by, g_by, gs_by
463
464   lsl g_by, g_by, #4
465   mov g_bx0, #0
466
467   add g_bx2, g_bx, g_bx
468   add g_bx3, g_bx, g_bx2
469
470   stmia store_b, { g_bx0, g_bx, g_bx2, g_bx3, b_base, g_by }
471
472   ldmia sp!, { r4 - r11, pc }
473
474
475 #define psx_gpu                                  r0
476 #define v_a                                      r1
477 #define v_b                                      r2
478 #define v_c                                      r3
479
480 #define temp                                     r14
481
482 #define x_a                                      r4
483 #define x_b                                      r5
484 #define x_c                                      r6
485 #define y_a                                      r1
486 #define y_b                                      r2
487 #define y_c                                      r3
488
489 #define height_minor_a                           r7
490 #define height_minor_b                           r8
491 #define height_major                             r9
492 #define height                                   r9
493
494 #define reciprocal_table_ptr                     r10
495
496 #define edge_alt_low                             r4
497 #define edge_alt_high                            r5
498 #define edge_dx_dy_alt                           r6
499 #define edge_shift_alt                           r10
500
501 #define edge_dx_dy_alt_low                       r4
502 #define edge_dx_dy_alt_high                      r5
503
504 #define span_edge_data                           r4
505 #define span_uvrg_offset                         r5
506 #define span_b_offset                            r6
507
508 #define clip                                     r14
509
510 #define b                                        r11
511 #define b_dy                                     r12
512
513
514 #define alternate_x                              q0
515 #define alternate_dx_dy                          q1
516 #define alternate_x_32                           q2
517
518 #define alternate_x_low                          d0
519 #define alternate_x_high                         d1
520 #define alternate_dx_dy_low                      d2
521 #define alternate_dx_dy_high                     d3
522 #define alternate_x_32_low                       d4
523 #define alternate_x_32_high                      d5
524
525 #define left_x                                   q3
526 #define right_x                                  q4
527 #define left_dx_dy                               q5
528 #define right_dx_dy                              q6
529 #define left_edge                                q7
530 #define right_edge                               q8
531
532 #define left_x_low                               d6
533 #define left_x_high                              d7
534 #define right_x_low                              d8
535 #define right_x_high                             d9
536 #define left_dx_dy_low                           d10
537 #define left_dx_dy_high                          d11
538 #define right_dx_dy_low                          d12
539 #define right_dx_dy_high                         d13
540 #define left_edge_low                            d14
541 #define left_edge_high                           d15
542 #define right_edge_low                           d16
543 #define right_edge_high                          d17
544
545 #define y_mid_point                              d18
546 #define c_0x0004                                 d19
547
548 #define left_right_x_16                          q11
549 #define span_shifts_y                            q12
550 #define c_0x0001                                 q13
551
552 #define span_shifts                              d24
553 #define y_x4                                     d25
554 #define c_0xFFFE                                 d26
555 #define c_0x0007                                 d27
556
557 #define left_right_x_16_low                      d22
558 #define left_right_x_16_high                     d23
559
560 #define uvrg                                     q14
561 #define uvrg_dy                                  q15
562
563 #define alternate_x_16                           d4
564
565 #define v_clip                                   q3
566 #define v_clip_low                               d6
567
568 #define right_x_32                               q10
569 #define left_x_32                                q11
570 #define alternate_select                         d24
571
572 #define right_x_32_low                           d20
573 #define right_x_32_high                          d21
574 #define left_x_32_low                            d22
575 #define left_x_32_high                           d23
576
577 #define edges_xy                                 q0
578 #define edges_dx_dy                              d2
579 #define edge_shifts                              d3
580 #define edge_shifts_64                           q2
581
582 #define edges_xy_left                            d0
583 #define edges_xy_right                           d1
584
585 #define height_reciprocals                       d6
586 #define heights                                  d7
587
588 #define widths                                   d8
589 #define c_0x01                                   d9
590 #define x_starts                                 d10
591 #define x_ends                                   d11
592
593 #define heights_b                                d12
594 #define edges_dx_dy_64                           q10
595
596 #define edges_dx_dy_64_left                      d20
597 #define edges_dx_dy_64_right                     d21
598
599
600 #define setup_spans_prologue()                                                 \
601   stmdb sp!, { r4 - r11, lr };                                                 \
602                                                                                \
603   ldrsh x_a, [ v_a, #8 ];                                                      \
604   ldrsh x_b, [ v_b, #8 ];                                                      \
605   ldrsh x_c, [ v_c, #8 ];                                                      \
606   ldrsh y_a, [ v_a, #10 ];                                                     \
607   ldrsh y_b, [ v_b, #10 ];                                                     \
608   ldrsh y_c, [ v_c, #10 ];                                                     \
609                                                                                \
610   add temp, psx_gpu, #psx_gpu_uvrg_offset;                                     \
611   vld1.32 { uvrg }, [ temp ];                                                  \
612   add temp, psx_gpu, #psx_gpu_uvrg_dy_offset;                                  \
613   vld1.32 { uvrg_dy }, [ temp ];                                               \
614   movw reciprocal_table_ptr, :lower16:reciprocal_table;                        \
615   movt reciprocal_table_ptr, :upper16:reciprocal_table;                        \
616                                                                                \
617   vmov.u32 c_0x01, #0x01                                                       \
618
619 #define setup_spans_load_b()                                                   \
620   ldr b, [ psx_gpu, #psx_gpu_b_offset ];                                       \
621   ldr b_dy, [ psx_gpu, #psx_gpu_b_dy_offset ]                                  \
622
623 #define setup_spans_prologue_b()                                               \
624   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
625   add temp, psx_gpu, #psx_gpu_viewport_start_x_offset;                         \
626                                                                                \
627   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
628   vmov.u16 c_0x0004, #0x0004;                                                  \
629                                                                                \
630   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
631   vmov.u16 c_0x0001, #0x0001;                                                  \
632                                                                                \
633   vld1.u16 { left_edge_low[], left_edge_high[] }, [ temp ];                    \
634   add temp, psx_gpu, #psx_gpu_viewport_end_x_offset;                           \
635                                                                                \
636   vld1.u16 { right_edge_low[], right_edge_high[] }, [ temp ];                  \
637   vadd.u16 right_edge, right_edge, c_0x0001;                                   \
638                                                                                \
639   vmov.u16 c_0x0007, #0x0007;                                                  \
640   vmvn.u16 c_0xFFFE, #0x0001                                                   \
641
642
643 #define compute_edge_delta_x2()                                                \
644   ldr temp, [ reciprocal_table_ptr, height, lsl #2 ];                          \
645                                                                                \
646   vdup.u32 heights, height;                                                    \
647   vsub.u32 widths, x_ends, x_starts;                                           \
648                                                                                \
649   vdup.u32 edge_shifts, temp;                                                  \
650   vsub.u32 heights_b, heights, c_0x01;                                         \
651   vshr.u32 height_reciprocals, edge_shifts, #12;                               \
652                                                                                \
653   vmla.s32 heights_b, x_starts, heights;                                       \
654   vbic.u16 edge_shifts, #0xE0;                                                 \
655   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
656   vmull.s32 edges_xy, heights_b, height_reciprocals                            \
657
658 #define width_alt                 r6
659 #define height_reciprocal_alt     r11
660 #define height_b_alt              r12
661
662 #define compute_edge_delta_x3(start_c, height_a, height_b)                     \
663   vmov.u32 heights, height_a, height_b;                                        \
664   ldr temp, [ reciprocal_table_ptr, height_a, lsl #2 ];                        \
665   vmov.u32 edge_shifts[0], temp;                                               \
666   ldr temp, [ reciprocal_table_ptr, height_b, lsl #2 ];                        \
667   vmov.u32 edge_shifts[1], temp;                                               \
668   ldr edge_shift_alt, [ reciprocal_table_ptr, height_minor_b, lsl #2 ];        \
669                                                                                \
670   vsub.u32 widths, x_ends, x_starts;                                           \
671   sub width_alt, x_c, start_c;                                                 \
672                                                                                \
673   vsub.u32 heights_b, heights, c_0x01;                                         \
674   sub height_b_alt, height_minor_b, #1;                                        \
675                                                                                \
676   vshr.u32 height_reciprocals, edge_shifts, #12;                               \
677   lsr height_reciprocal_alt, edge_shift_alt, #12;                              \
678                                                                                \
679   vmla.s32 heights_b, x_starts, heights;                                       \
680   mla height_b_alt, height_minor_b, start_c, height_b_alt;                     \
681                                                                                \
682   vbic.u16 edge_shifts, #0xE0;                                                 \
683   and edge_shift_alt, edge_shift_alt, #0x1F;                                   \
684                                                                                \
685   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
686   mul edge_dx_dy_alt, width_alt, height_reciprocal_alt;                        \
687                                                                                \
688   vmull.s32 edges_xy, heights_b, height_reciprocals;                           \
689   smull edge_alt_low, edge_alt_high, height_b_alt, height_reciprocal_alt       \
690
691
692 #define setup_spans_adjust_y_up()                                              \
693   vsub.u32 y_x4, y_x4, c_0x0004                                                \
694
695 #define setup_spans_adjust_y_down()                                            \
696   vadd.u32 y_x4, y_x4, c_0x0004                                                \
697
698 #define setup_spans_adjust_interpolants_up()                                   \
699   vsub.u32 uvrg, uvrg, uvrg_dy;                                                \
700   sub b, b, b_dy                                                               \
701
702 #define setup_spans_adjust_interpolants_down()                                 \
703   vadd.u32 uvrg, uvrg, uvrg_dy;                                                \
704   add b, b, b_dy                                                               \
705
706
707 #define setup_spans_clip_interpolants_increment()                              \
708   mla b, b_dy, clip, b;                                                        \
709   vmla.s32 uvrg, uvrg_dy, v_clip                                               \
710
711 #define setup_spans_clip_interpolants_decrement()                              \
712   mls b, b_dy, clip, b;                                                        \
713   vmls.s32 uvrg, uvrg_dy, v_clip                                               \
714
715 #define setup_spans_clip_alternate_yes()                                       \
716   smlal edge_alt_low, edge_alt_high, edge_dx_dy_alt, clip                      \
717
718 #define setup_spans_clip_alternate_no()                                        \
719
720 #define setup_spans_clip(direction, alternate_active)                          \
721   vdup.u32 v_clip, clip;                                                       \
722   setup_spans_clip_alternate_##alternate_active();                             \
723   setup_spans_clip_interpolants_##direction();                                 \
724   vmlal.s32 edges_xy, edges_dx_dy, v_clip_low                                  \
725
726
727 #define setup_spans_adjust_edges_alternate_no(left_index, right_index)         \
728   vmovl.s32 edge_shifts_64, edge_shifts;                                       \
729   vmovl.s32 edges_dx_dy_64, edges_dx_dy;                                       \
730                                                                                \
731   vshl.s64 edges_xy, edges_xy, edge_shifts_64;                                 \
732   vshl.s64 edges_dx_dy_64, edges_dx_dy_64, edge_shifts_64;                     \
733                                                                                \
734   vmov left_x_low, edges_xy_##left_index;                                      \
735   vmov right_x_low, edges_xy_##right_index;                                    \
736                                                                                \
737   vmov left_dx_dy_low, edges_dx_dy_64_##left_index;                            \
738   vmov left_dx_dy_high, edges_dx_dy_64_##left_index;                           \
739   vmov right_dx_dy_low, edges_dx_dy_64_##right_index;                          \
740   vmov right_dx_dy_high, edges_dx_dy_64_##right_index;                         \
741                                                                                \
742   vadd.u64 left_x_high, left_x_low, left_dx_dy_low;                            \
743   vadd.u64 right_x_high, right_x_low, right_dx_dy_low;                         \
744                                                                                \
745   vadd.u64 left_dx_dy, left_dx_dy, left_dx_dy;                                 \
746   vadd.u64 right_dx_dy, right_dx_dy, right_dx_dy                               \
747
748
749 #define setup_spans_adjust_edges_alternate_yes(left_index, right_index)        \
750   setup_spans_adjust_edges_alternate_no(left_index, right_index);              \
751                                                                                \
752   vdup.u16 y_mid_point, y_b;                                                   \
753   rsb temp, edge_shift_alt, #32;                                               \
754                                                                                \
755   lsl edge_alt_high, edge_alt_high, edge_shift_alt;                            \
756   orr edge_alt_high, edge_alt_high, edge_alt_low, lsr temp;                    \
757   lsl edge_alt_low, edge_alt_low, edge_shift_alt;                              \
758   vmov alternate_x_low, edge_alt_low, edge_alt_high;                           \
759                                                                                \
760   asr edge_dx_dy_alt_high, edge_dx_dy_alt, temp;                               \
761   lsl edge_dx_dy_alt_low, edge_dx_dy_alt, edge_shift_alt;                      \
762   vmov alternate_dx_dy_low, edge_dx_dy_alt_low, edge_dx_dy_alt_high;           \
763   vmov alternate_dx_dy_high, alternate_dx_dy_low;                              \
764                                                                                \
765   vadd.u64 alternate_x_high, alternate_x_low, alternate_dx_dy_low;             \
766   vadd.u64 alternate_dx_dy, alternate_dx_dy, alternate_dx_dy                   \
767
768
769 #define setup_spans_y_select_up()                                              \
770   vclt.s16 alternate_select, y_x4, y_mid_point                                 \
771
772 #define setup_spans_y_select_down()                                            \
773   vcgt.s16 alternate_select, y_x4, y_mid_point                                 \
774
775
776 #define setup_spans_alternate_select_left()                                    \
777   vbit.u16 left_right_x_16_low, alternate_x_16, alternate_select               \
778
779 #define setup_spans_alternate_select_right()                                   \
780   vbit.u16 left_right_x_16_high, alternate_x_16, alternate_select              \
781
782
783 #define setup_spans_set_x4_alternate_yes(alternate, direction)                 \
784   vshrn.s64 alternate_x_32_low, alternate_x, #32;                              \
785   vshrn.s64 left_x_32_low, left_x, #32;                                        \
786   vshrn.s64 right_x_32_low, right_x, #32;                                      \
787                                                                                \
788   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
789   vadd.u64 left_x, left_x, left_dx_dy;                                         \
790   vadd.u64 right_x, right_x, right_dx_dy;                                      \
791                                                                                \
792   vshrn.s64 alternate_x_32_high, alternate_x, #32;                             \
793   vshrn.s64 left_x_32_high, left_x, #32;                                       \
794   vshrn.s64 right_x_32_high, right_x, #32;                                     \
795                                                                                \
796   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
797   vadd.u64 left_x, left_x, left_dx_dy;                                         \
798   vadd.u64 right_x, right_x, right_dx_dy;                                      \
799                                                                                \
800   vmovn.u32 alternate_x_16, alternate_x_32;                                    \
801   setup_spans_y_select_##direction();                                          \
802   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
803                                                                                \
804   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
805   setup_spans_alternate_select_##alternate();                                  \
806                                                                                \
807   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
808   str b, [ span_b_offset ], #4;                                                \
809   setup_spans_adjust_interpolants_##direction();                               \
810                                                                                \
811   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
812                                                                                \
813   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
814   str b, [ span_b_offset ], #4;                                                \
815   setup_spans_adjust_interpolants_##direction();                               \
816                                                                                \
817   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
818                                                                                \
819   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
820   str b, [ span_b_offset ], #4;                                                \
821   setup_spans_adjust_interpolants_##direction();                               \
822                                                                                \
823   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
824   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
825   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
826                                                                                \
827   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
828   str b, [ span_b_offset ], #4;                                                \
829   setup_spans_adjust_interpolants_##direction();                               \
830                                                                                \
831   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
832   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
833                                                                                \
834   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
835                                                                                \
836   setup_spans_adjust_y_##direction()                                           \
837
838
839 #define setup_spans_set_x4_alternate_no(alternate, direction)                  \
840   vshrn.s64 left_x_32_low, left_x, #32;                                        \
841   vshrn.s64 right_x_32_low, right_x, #32;                                      \
842                                                                                \
843   vadd.u64 left_x, left_x, left_dx_dy;                                         \
844   vadd.u64 right_x, right_x, right_dx_dy;                                      \
845                                                                                \
846   vshrn.s64 left_x_32_high, left_x, #32;                                       \
847   vshrn.s64 right_x_32_high, right_x, #32;                                     \
848                                                                                \
849   vadd.u64 left_x, left_x, left_dx_dy;                                         \
850   vadd.u64 right_x, right_x, right_dx_dy;                                      \
851                                                                                \
852   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
853   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
854                                                                                \
855   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
856   str b, [ span_b_offset ], #4;                                                \
857   setup_spans_adjust_interpolants_##direction();                               \
858                                                                                \
859   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
860                                                                                \
861   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
862   str b, [ span_b_offset ], #4;                                                \
863   setup_spans_adjust_interpolants_##direction();                               \
864                                                                                \
865   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
866                                                                                \
867   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
868   str b, [ span_b_offset ], #4;                                                \
869   setup_spans_adjust_interpolants_##direction();                               \
870                                                                                \
871   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
872   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
873   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
874                                                                                \
875   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
876   str b, [ span_b_offset ], #4;                                                \
877   setup_spans_adjust_interpolants_##direction();                               \
878                                                                                \
879   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
880   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
881                                                                                \
882   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
883                                                                                \
884   setup_spans_adjust_y_##direction()                                           \
885
886
887 #define edge_adjust_low           r11
888 #define edge_adjust_high          r12
889
890 #define setup_spans_alternate_adjust_yes()                                     \
891   smull edge_adjust_low, edge_adjust_high, edge_dx_dy_alt, height_minor_a;     \
892   subs edge_alt_low, edge_alt_low, edge_adjust_low;                            \
893   sbc edge_alt_high, edge_alt_high, edge_adjust_high                           \
894
895 #define setup_spans_alternate_adjust_no()                                      \
896
897
898 #define setup_spans_down(left_index, right_index, alternate, alternate_active) \
899   setup_spans_alternate_adjust_##alternate_active();                           \
900   setup_spans_load_b();                                                        \
901                                                                                \
902   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                     \
903   subs y_c, y_c, temp;                                                         \
904   subgt height, height, y_c;                                                   \
905   addgt height, height, #1;                                                    \
906                                                                                \
907   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                   \
908   subs clip, temp, y_a;                                                        \
909   ble 0f;                                                                      \
910                                                                                \
911   sub height, height, clip;                                                    \
912   add y_a, y_a, clip;                                                          \
913   setup_spans_clip(increment, alternate_active);                               \
914                                                                                \
915  0:                                                                            \
916   cmp height, #0;                                                              \
917   ble 1f;                                                                      \
918                                                                                \
919   orr temp, y_a, y_a, lsl #16;                                                 \
920   add temp, temp, #(1 << 16);                                                  \
921   add y_a, temp, #2;                                                           \
922   add y_a, y_a, #(2 << 16);                                                    \
923   vmov.u32 y_x4, temp, y_a;                                                    \
924                                                                                \
925   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
926    right_index);                                                               \
927   setup_spans_prologue_b();                                                    \
928                                                                                \
929   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
930                                                                                \
931  2:                                                                            \
932   setup_spans_set_x4_alternate_##alternate_active(alternate, down);            \
933   subs height, height, #4;                                                     \
934   bhi 2b;                                                                      \
935                                                                                \
936  1:                                                                            \
937
938
939 #define setup_spans_alternate_pre_increment_yes()                              \
940   adds edge_alt_low, edge_alt_low, edge_dx_dy_alt;                             \
941   adc edge_alt_high, edge_alt_high, edge_dx_dy_alt, asr #31                    \
942
943 #define setup_spans_alternate_pre_increment_no()                               \
944
945
946 #define setup_spans_up_decrement_yes()                                         \
947   suble height, height, #1                                                     \
948
949 #define setup_spans_up_decrement_no()                                          \
950
951
952 #define setup_spans_up(left_index, right_index, alternate, alternate_active)   \
953   setup_spans_alternate_adjust_##alternate_active();                           \
954   setup_spans_load_b();                                                        \
955   sub y_a, y_a, #1;                                                            \
956                                                                                \
957   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                    \
958   subs temp, temp, y_c;                                                        \
959   subgt height, height, temp;                                                  \
960   setup_spans_up_decrement_##alternate_active();                               \
961                                                                                \
962   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                      \
963   subs clip, y_a, temp;                                                        \
964   ble 0f;                                                                      \
965                                                                                \
966   sub height, height, clip;                                                    \
967   sub y_a, y_a, clip;                                                          \
968   setup_spans_clip(decrement, alternate_active);                               \
969                                                                                \
970  0:                                                                            \
971   cmp height, #0;                                                              \
972   ble 1f;                                                                      \
973                                                                                \
974   orr temp, y_a, y_a, lsl #16;                                                 \
975   sub temp, temp, #(1 << 16);                                                  \
976   sub y_a, temp, #2;                                                           \
977   sub y_a, y_a, #(2 << 16);                                                    \
978   vmov.u32 y_x4, temp, y_a;                                                    \
979                                                                                \
980   vaddw.s32 edges_xy, edges_xy, edges_dx_dy;                                   \
981                                                                                \
982   setup_spans_alternate_pre_increment_##alternate_active();                    \
983   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
984    right_index);                                                               \
985   setup_spans_adjust_interpolants_up();                                        \
986   setup_spans_prologue_b();                                                    \
987                                                                                \
988   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
989                                                                                \
990  2:                                                                            \
991   setup_spans_set_x4_alternate_##alternate_active(alternate, up);              \
992   subs height, height, #4;                                                     \
993   bhi 2b;                                                                      \
994                                                                                \
995  1:                                                                            \
996
997
998 #define setup_spans_epilogue()                                                 \
999   ldmia sp!, { r4 - r11, pc }                                                  \
1000
1001
1002 #define setup_spans_up_up(minor, major)                                        \
1003   setup_spans_prologue();                                                      \
1004   sub height_minor_a, y_a, y_b;                                                \
1005   sub height_minor_b, y_b, y_c;                                                \
1006   sub height, y_a, y_c;                                                        \
1007                                                                                \
1008   vdup.u32 x_starts, x_a;                                                      \
1009   vmov.u32 x_ends, x_c, x_b;                                                   \
1010                                                                                \
1011   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
1012   setup_spans_up(major, minor, minor, yes);                                    \
1013   setup_spans_epilogue()                                                       \
1014
1015 function(setup_spans_up_left)
1016   setup_spans_up_up(left, right)
1017
1018 function(setup_spans_up_right)
1019   setup_spans_up_up(right, left)
1020
1021
1022 #define setup_spans_down_down(minor, major)                                    \
1023   setup_spans_prologue();                                                      \
1024   sub height_minor_a, y_b, y_a;                                                \
1025   sub height_minor_b, y_c, y_b;                                                \
1026   sub height, y_c, y_a;                                                        \
1027                                                                                \
1028   vdup.u32 x_starts, x_a;                                                      \
1029   vmov.u32 x_ends, x_c, x_b;                                                   \
1030                                                                                \
1031   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
1032   setup_spans_down(major, minor, minor, yes);                                  \
1033   setup_spans_epilogue()                                                       \
1034
1035 function(setup_spans_down_left)
1036   setup_spans_down_down(left, right)
1037
1038 function(setup_spans_down_right)
1039   setup_spans_down_down(right, left)
1040
1041
1042 #define setup_spans_up_flat()                                                  \
1043   sub height, y_a, y_c;                                                        \
1044                                                                                \
1045   compute_edge_delta_x2();                                                     \
1046   setup_spans_up(left, right, none, no);                                       \
1047   setup_spans_epilogue()                                                       \
1048
1049 function(setup_spans_up_a)
1050   setup_spans_prologue()
1051
1052   vmov.u32 x_starts, x_a, x_b
1053   vdup.u32 x_ends, x_c
1054
1055   setup_spans_up_flat()
1056
1057 function(setup_spans_up_b)
1058   setup_spans_prologue()
1059
1060   vdup.u32 x_starts, x_a
1061   vmov.u32 x_ends, x_b, x_c
1062
1063   setup_spans_up_flat()
1064
1065 #define setup_spans_down_flat()                                                \
1066   sub height, y_c, y_a;                                                        \
1067                                                                                \
1068   compute_edge_delta_x2();                                                     \
1069   setup_spans_down(left, right, none, no);                                     \
1070   setup_spans_epilogue()                                                       \
1071
1072 function(setup_spans_down_a)
1073   setup_spans_prologue()
1074
1075   vmov.u32 x_starts, x_a, x_b
1076   vdup.u32 x_ends, x_c
1077
1078   setup_spans_down_flat()
1079
1080 function(setup_spans_down_b)
1081   setup_spans_prologue()
1082
1083   vdup.u32 x_starts, x_a
1084   vmov.u32 x_ends, x_b, x_c
1085
1086   setup_spans_down_flat()
1087
1088
1089 #define middle_y                                          r9
1090
1091 #define edges_xy_b                                        q11
1092 #define edges_dx_dy_b                                     d26
1093 #define edge_shifts_b                                     d27
1094 #define edges_dx_dy_and_shifts_b                          q13
1095 #define height_increment                                  d20
1096
1097 #define edges_dx_dy_and_shifts                            q1
1098
1099 #define edges_xy_b_left                                   d22
1100 #define edges_xy_b_right                                  d23
1101
1102 #define setup_spans_up_down_load_edge_set_b()                                  \
1103   vmov edges_xy, edges_xy_b;                                                   \
1104   vmov edges_dx_dy_and_shifts, edges_dx_dy_and_shifts_b                        \
1105
1106
1107 function(setup_spans_up_down)
1108   setup_spans_prologue()
1109
1110   // s32 middle_y = y_a;
1111   sub height_minor_a, y_a, y_b
1112   sub height_minor_b, y_c, y_a
1113   sub height_major, y_c, y_b
1114
1115   vmov.u32 x_starts, x_a, x_c
1116   vdup.u32 x_ends, x_b
1117
1118   compute_edge_delta_x3(x_a, height_minor_a, height_major)
1119
1120   mov temp, #0
1121   vmov.u32 height_increment, temp, height_minor_b
1122   vmlal.s32 edges_xy, edges_dx_dy, height_increment
1123
1124   vmov edges_xy_b_left, edge_alt_low, edge_alt_high
1125   vmov edges_xy_b_right, edges_xy_right
1126
1127   vmov edge_shifts_b, edge_shifts
1128   vmov.u32 edge_shifts_b[0], edge_shift_alt
1129
1130   vneg.s32 edges_dx_dy_b, edges_dx_dy
1131   vmov.u32 edges_dx_dy_b[0], edge_dx_dy_alt
1132
1133   mov middle_y, y_a
1134   
1135   setup_spans_load_b()
1136   sub y_a, y_a, #1
1137
1138   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1139   subs temp, temp, y_b
1140   subgt height_minor_a, height_minor_a, temp
1141
1142   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1143   subs clip, y_a, temp
1144   ble 0f
1145
1146   sub height_minor_a, height_minor_a, clip
1147   sub y_a, y_a, clip
1148   setup_spans_clip(decrement, no)
1149
1150  0:                                                                
1151   cmp height_minor_a, #0
1152   ble 3f
1153
1154   orr temp, y_a, y_a, lsl #16
1155   sub temp, temp, #(1 << 16)
1156   sub y_a, temp, #2
1157   sub y_a, y_a, #(2 << 16)
1158   vmov.u32 y_x4, temp, y_a
1159
1160   vaddw.s32 edges_xy, edges_xy, edges_dx_dy
1161
1162   strh height_minor_a, [ psx_gpu, #psx_gpu_num_spans_offset ]
1163
1164   setup_spans_adjust_edges_alternate_no(left, right); 
1165   setup_spans_adjust_interpolants_up()
1166   setup_spans_up_down_load_edge_set_b()
1167
1168   setup_spans_prologue_b()
1169
1170
1171  2: 
1172   setup_spans_set_x4_alternate_no(none, up)
1173   subs height_minor_a, height_minor_a, #4
1174   bhi 2b
1175
1176   add span_edge_data, span_edge_data, height_minor_a, lsl #3
1177   add span_uvrg_offset, span_uvrg_offset, height_minor_a, lsl #4
1178   add span_b_offset, span_b_offset, height_minor_a, lsl #2
1179
1180  4:
1181   add temp, psx_gpu, #psx_gpu_uvrg_offset
1182   vld1.32 { uvrg }, [ temp ]
1183   mov y_a, middle_y
1184   
1185   setup_spans_load_b()
1186
1187   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1188   subs y_c, y_c, temp
1189   subgt height_minor_b, height_minor_b, y_c
1190   addgt height_minor_b, height_minor_b, #1
1191
1192   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1193   subs clip, temp, y_a
1194   ble 0f
1195
1196   sub height_minor_b, height_minor_b, clip
1197   add y_a, y_a, clip
1198   setup_spans_clip(increment, no)
1199
1200  0:
1201   cmp height_minor_b, #0
1202   ble 1f
1203
1204   orr temp, y_a, y_a, lsl #16
1205   add temp, temp, #(1 << 16) 
1206   add y_a, temp, #2
1207   add y_a, y_a, #(2 << 16)
1208   vmov.u32 y_x4, temp, y_a
1209
1210   setup_spans_adjust_edges_alternate_no(left, right)
1211
1212   ldrh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1213   add temp, temp, height_minor_b
1214   strh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1215
1216  2:                                                     
1217   setup_spans_set_x4_alternate_no(none, down)
1218   subs height_minor_b, height_minor_b, #4
1219   bhi 2b
1220
1221  1:
1222   setup_spans_epilogue()
1223
1224  3:
1225   setup_spans_up_down_load_edge_set_b()
1226   setup_spans_prologue_b()
1227   bal 4b
1228
1229
1230 #undef span_uvrg_offset
1231 #undef span_edge_data
1232 #undef span_b_offset
1233 #undef left_x
1234 #undef b
1235
1236 #define psx_gpu                                  r0
1237 #define num_spans                                r1
1238 #define span_uvrg_offset                         r2
1239 #define span_edge_data                           r3
1240 #define span_b_offset                            r4
1241 #define b_dx                                     r5
1242 #define span_num_blocks                          r6
1243 #define y                                        r7
1244 #define left_x                                   r8
1245 #define b                                        r9
1246 #define dither_offset_ptr                        r10
1247 #define block_ptr_a                              r11
1248 #define fb_ptr                                   r12
1249 #define num_blocks                               r14
1250
1251 #define uvrg_dx_ptr                              r2
1252 #define texture_mask_ptr                         r3
1253 #define dither_shift                             r8
1254 #define dither_row                               r10
1255
1256 #define c_32                                     r7
1257 #define b_dx4                                    r8
1258 #define b_dx8                                    r9
1259 #define block_ptr_b                              r10
1260
1261 #define block_span_ptr                           r10
1262 #define right_mask                               r8
1263
1264 #define color                                    r2
1265 #define color_r                                  r3
1266 #define color_g                                  r4
1267 #define color_b                                  r5
1268
1269 #undef uvrg
1270
1271 #define u_block                                  q0
1272 #define v_block                                  q1
1273 #define r_block                                  q2
1274 #define g_block                                  q3
1275 #define b_block                                  q4
1276
1277 #define uv_dx4                                   d10
1278 #define rg_dx4                                   d11
1279 #define uv_dx8                                   d12
1280 #define rg_dx8                                   d13
1281 #define b_whole_8                                d14
1282 #define fb_mask_ptrs                             d15
1283
1284 #define uvrg_dx4                                 q5
1285 #define uvrg_dx8                                 q6
1286 #define uv_dx8                                   d12
1287 #define rg_dx8                                   d13
1288
1289 #define u_whole                                  q8
1290 #define v_whole                                  q9
1291 #define r_whole                                  q10
1292 #define g_whole                                  q11
1293 #define b_whole                                  q12
1294
1295 #define u_whole_low                              d16
1296 #define u_whole_high                             d17
1297 #define v_whole_low                              d18
1298 #define v_whole_high                             d19
1299 #define r_whole_low                              d20
1300 #define r_whole_high                             d21
1301 #define g_whole_low                              d22
1302 #define g_whole_high                             d23
1303 #define b_whole_low                              d24
1304 #define b_whole_high                             d25
1305
1306 #define dx4                                      q13
1307 #define dx8                                      q13
1308
1309 #define u_whole_8                                d26
1310 #define v_whole_8                                d27
1311 #define u_whole_8b                               d24
1312 #define r_whole_8                                d24
1313 #define g_whole_8                                d25
1314
1315 #define uv_whole_8                               q13
1316 #define uv_whole_8b                              q14
1317
1318 #define dither_offsets                           q14
1319 #define texture_mask                             q15
1320 #define texture_mask_u                           d30
1321 #define texture_mask_v                           d31
1322
1323 #define dither_offsets_short                     d28
1324
1325 #define v_left_x                                 q8
1326 #define uvrg                                     q9
1327 #define block_span                               q10
1328
1329 #define uv                                       d18
1330 #define rg                                       d19
1331
1332 #define draw_mask                                q1
1333 #define draw_mask_edge                           q13
1334 #define test_mask                                q0
1335
1336 #define uvrg_dx                                  q3
1337
1338 #define colors                                   q2
1339
1340 #define setup_blocks_texture_swizzled()                                        \
1341   vand.u8 u_whole_8b, u_whole_8, texture_mask_u;                               \
1342   vsli.u8 u_whole_8, v_whole_8, #4;                                            \
1343   vsri.u8 v_whole_8, u_whole_8b, #4                                            \
1344
1345 #define setup_blocks_texture_unswizzled()                                      \
1346
1347
1348 #define setup_blocks_shaded_textured_builder(swizzling)                        \
1349 .align 3;                                                                      \
1350                                                                                \
1351 function(setup_blocks_shaded_textured_dithered_##swizzling##_indirect)         \
1352   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1353   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1354                                                                                \
1355   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1356   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1357                                                                                \
1358   cmp num_spans, #0;                                                           \
1359   bxeq lr;                                                                     \
1360                                                                                \
1361   stmdb sp!, { r4 - r11, r14 };                                                \
1362   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1363                                                                                \
1364   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
1365   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1366                                                                                \
1367   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1368   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1369                                                                                \
1370   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1371   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1372                                                                                \
1373   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
1374   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1375                                                                                \
1376   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1377                                                                                \
1378  0:                                                                            \
1379   vmov.u8 fb_mask_ptrs, #0;                                                    \
1380                                                                                \
1381   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1382   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1383                                                                                \
1384   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1385   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1386                                                                                \
1387   cmp span_num_blocks, #0;                                                     \
1388   beq 1f;                                                                      \
1389                                                                                \
1390   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1391   add num_blocks, span_num_blocks, num_blocks;                                 \
1392                                                                                \
1393   cmp num_blocks, #MAX_BLOCKS;                                                 \
1394   bgt 2f;                                                                      \
1395                                                                                \
1396  3:                                                                            \
1397   ldr b, [ span_b_offset ];                                                    \
1398   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1399                                                                                \
1400   vdup.u32 v_left_x, left_x;                                                   \
1401   and y, y, #0x3;                                                              \
1402                                                                                \
1403   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1404   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1405                                                                                \
1406   mla b, b_dx, left_x, b;                                                      \
1407   and dither_shift, left_x, #0x03;                                             \
1408                                                                                \
1409   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1410   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1411                                                                                \
1412   mov dither_shift, dither_shift, lsl #3;                                      \
1413   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1414                                                                                \
1415   mov c_32, #32;                                                               \
1416   subs span_num_blocks, span_num_blocks, #1;                                   \
1417                                                                                \
1418   mov dither_row, dither_row, ror dither_shift;                                \
1419   mov b_dx4, b_dx, lsl #2;                                                     \
1420                                                                                \
1421   vdup.u32 dither_offsets_short, dither_row;                                   \
1422   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1423                                                                                \
1424   vdup.u32 b_block, b;                                                         \
1425   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1426                                                                                \
1427   vdup.u32 u_block, uv[0];                                                     \
1428   mov b_dx8, b_dx, lsl #3;                                                     \
1429                                                                                \
1430   vdup.u32 v_block, uv[1];                                                     \
1431   vdup.u32 r_block, rg[0];                                                     \
1432   vdup.u32 g_block, rg[1];                                                     \
1433                                                                                \
1434   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1435                                                                                \
1436   vadd.u32 u_block, u_block, block_span;                                       \
1437   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1438                                                                                \
1439   vadd.u32 v_block, v_block, block_span;                                       \
1440   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1441                                                                                \
1442   vadd.u32 r_block, r_block, block_span;                                       \
1443   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1444                                                                                \
1445   vadd.u32 g_block, g_block, block_span;                                       \
1446   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
1447                                                                                \
1448   vadd.u32 b_block, b_block, block_span;                                       \
1449   add block_ptr_b, block_ptr_a, #16;                                           \
1450                                                                                \
1451   vshrn.u32 u_whole_low, u_block, #16;                                         \
1452   vshrn.u32 v_whole_low, v_block, #16;                                         \
1453   vshrn.u32 r_whole_low, r_block, #16;                                         \
1454   vshrn.u32 g_whole_low, g_block, #16;                                         \
1455                                                                                \
1456   vdup.u32 dx4, uv_dx4[0];                                                     \
1457   vshrn.u32 b_whole_low, b_block, #16;                                         \
1458                                                                                \
1459   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1460   vdup.u32 dx4, uv_dx4[1];                                                     \
1461                                                                                \
1462   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1463   vdup.u32 dx4, rg_dx4[0];                                                     \
1464                                                                                \
1465   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1466   vdup.u32 dx4, rg_dx4[1];                                                     \
1467                                                                                \
1468   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1469   vdup.u32 dx4, b_dx4;                                                         \
1470                                                                                \
1471   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1472   vdup.u32 dx8, uv_dx8[0];                                                     \
1473                                                                                \
1474   vadd.u32 u_block, u_block, dx8;                                              \
1475   vdup.u32 dx8, uv_dx8[1];                                                     \
1476                                                                                \
1477   vadd.u32 v_block, v_block, dx8;                                              \
1478   vdup.u32 dx8, rg_dx8[0];                                                     \
1479                                                                                \
1480   vadd.u32 r_block, r_block, dx8;                                              \
1481   vdup.u32 dx8, rg_dx8[1];                                                     \
1482                                                                                \
1483   vadd.u32 g_block, g_block, dx8;                                              \
1484   vdup.u32 dx8, b_dx8;                                                         \
1485                                                                                \
1486   vadd.u32 b_block, b_block, dx8;                                              \
1487   vmovn.u16 u_whole_8, u_whole;                                                \
1488                                                                                \
1489   vmovn.u16 v_whole_8, v_whole;                                                \
1490                                                                                \
1491   vmovn.u16 b_whole_8, b_whole;                                                \
1492   pld [ fb_ptr ];                                                              \
1493   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1494                                                                                \
1495   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1496   setup_blocks_texture_##swizzling();                                          \
1497                                                                                \
1498   vmovn.u16 r_whole_8, r_whole;                                                \
1499   beq 5f;                                                                      \
1500                                                                                \
1501  4:                                                                            \
1502   vmovn.u16 g_whole_8, g_whole;                                                \
1503   vshrn.u32 u_whole_low, u_block, #16;                                         \
1504                                                                                \
1505   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1506   vshrn.u32 v_whole_low, v_block, #16;                                         \
1507                                                                                \
1508   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1509   vshrn.u32 r_whole_low, r_block, #16;                                         \
1510                                                                                \
1511   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1512   vshrn.u32 g_whole_low, g_block, #16;                                         \
1513                                                                                \
1514   vdup.u32 dx4, uv_dx4[0];                                                     \
1515   vshrn.u32 b_whole_low, b_block, #16;                                         \
1516                                                                                \
1517   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1518   vdup.u32 dx4, uv_dx4[1];                                                     \
1519                                                                                \
1520   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1521   vdup.u32 dx4, rg_dx4[0];                                                     \
1522                                                                                \
1523   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1524   vdup.u32 dx4, rg_dx4[1];                                                     \
1525                                                                                \
1526   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1527   vdup.u32 dx4, b_dx4;                                                         \
1528                                                                                \
1529   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1530   vdup.u32 dx8, uv_dx8[0];                                                     \
1531                                                                                \
1532   vadd.u32 u_block, u_block, dx8;                                              \
1533   vdup.u32 dx8, uv_dx8[1];                                                     \
1534                                                                                \
1535   vadd.u32 v_block, v_block, dx8;                                              \
1536   vdup.u32 dx8, rg_dx8[0];                                                     \
1537                                                                                \
1538   vadd.u32 r_block, r_block, dx8;                                              \
1539   vdup.u32 dx8, rg_dx8[1];                                                     \
1540                                                                                \
1541   vadd.u32 g_block, g_block, dx8;                                              \
1542   vdup.u32 dx8, b_dx8;                                                         \
1543                                                                                \
1544   vadd.u32 b_block, b_block, dx8;                                              \
1545   vmovn.u16 u_whole_8, u_whole;                                                \
1546                                                                                \
1547   add fb_ptr, fb_ptr, #16;                                                     \
1548   vmovn.u16 v_whole_8, v_whole;                                                \
1549                                                                                \
1550   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1551   vmovn.u16 b_whole_8, b_whole;                                                \
1552                                                                                \
1553   pld [ fb_ptr ];                                                              \
1554                                                                                \
1555   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1556   subs span_num_blocks, span_num_blocks, #1;                                   \
1557                                                                                \
1558   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1559   setup_blocks_texture_##swizzling();                                          \
1560                                                                                \
1561   vmovn.u16 r_whole_8, r_whole;                                                \
1562   bne 4b;                                                                      \
1563                                                                                \
1564  5:                                                                            \
1565   vmovn.u16 g_whole_8, g_whole;                                                \
1566   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1567                                                                                \
1568   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1569   vdup.u8 draw_mask, right_mask;                                               \
1570                                                                                \
1571   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1572   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1573   vzip.u8 u_whole_8, v_whole_8;                                                \
1574                                                                                \
1575   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1576   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1577   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1578   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1579   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1580                                                                                \
1581  1:                                                                            \
1582   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1583   add span_b_offset, span_b_offset, #4;                                        \
1584                                                                                \
1585   add span_edge_data, span_edge_data, #8;                                      \
1586   subs num_spans, num_spans, #1;                                               \
1587                                                                                \
1588   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1589   bne 0b;                                                                      \
1590                                                                                \
1591   ldmia sp!, { r4 - r11, pc };                                                 \
1592                                                                                \
1593  2:                                                                            \
1594   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1595   vpush { texture_mask };                                                      \
1596   vpush { uvrg_dx4 };                                                          \
1597                                                                                \
1598   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1599   bl flush_render_block_buffer;                                                \
1600   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1601                                                                                \
1602   vpop { uvrg_dx4 };                                                           \
1603   vpop { texture_mask };                                                       \
1604                                                                                \
1605   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1606   vmov.u8 fb_mask_ptrs, #0;                                                    \
1607                                                                                \
1608   mov num_blocks, span_num_blocks;                                             \
1609   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1610   bal 3b                                                                       \
1611
1612
1613 setup_blocks_shaded_textured_builder(swizzled)
1614 setup_blocks_shaded_textured_builder(unswizzled)
1615
1616
1617 #define setup_blocks_unshaded_textured_builder(swizzling)                      \
1618 .align 3;                                                                      \
1619                                                                                \
1620 function(setup_blocks_unshaded_textured_dithered_##swizzling##_indirect)       \
1621   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1622   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1623                                                                                \
1624   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1625   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1626                                                                                \
1627   cmp num_spans, #0;                                                           \
1628   bxeq lr;                                                                     \
1629                                                                                \
1630   stmdb sp!, { r4 - r11, r14 };                                                \
1631   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1632                                                                                \
1633   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1634                                                                                \
1635   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1636   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1637                                                                                \
1638   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1639   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1640                                                                                \
1641   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1642                                                                                \
1643   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1644                                                                                \
1645  0:                                                                            \
1646   vmov.u8 fb_mask_ptrs, #0;                                                    \
1647                                                                                \
1648   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1649   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1650                                                                                \
1651   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1652   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1653                                                                                \
1654   cmp span_num_blocks, #0;                                                     \
1655   beq 1f;                                                                      \
1656                                                                                \
1657   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1658   add num_blocks, span_num_blocks, num_blocks;                                 \
1659                                                                                \
1660   cmp num_blocks, #MAX_BLOCKS;                                                 \
1661   bgt 2f;                                                                      \
1662                                                                                \
1663  3:                                                                            \
1664   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1665                                                                                \
1666   vdup.u32 v_left_x, left_x;                                                   \
1667   and y, y, #0x3;                                                              \
1668                                                                                \
1669   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1670   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1671                                                                                \
1672   and dither_shift, left_x, #0x03;                                             \
1673                                                                                \
1674   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1675   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1676                                                                                \
1677   mov dither_shift, dither_shift, lsl #3;                                      \
1678   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1679                                                                                \
1680   mov c_32, #32;                                                               \
1681   subs span_num_blocks, span_num_blocks, #1;                                   \
1682                                                                                \
1683   mov dither_row, dither_row, ror dither_shift;                                \
1684                                                                                \
1685   vdup.u32 dither_offsets_short, dither_row;                                   \
1686   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1687                                                                                \
1688   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1689                                                                                \
1690   vdup.u32 u_block, uv[0];                                                     \
1691                                                                                \
1692   vdup.u32 v_block, uv[1];                                                     \
1693   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1694                                                                                \
1695   vadd.u32 u_block, u_block, block_span;                                       \
1696   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1697                                                                                \
1698   vadd.u32 v_block, v_block, block_span;                                       \
1699   add block_ptr_b, block_ptr_a, #16;                                           \
1700                                                                                \
1701   vshrn.u32 u_whole_low, u_block, #16;                                         \
1702   vshrn.u32 v_whole_low, v_block, #16;                                         \
1703                                                                                \
1704   vdup.u32 dx4, uv_dx4[0];                                                     \
1705                                                                                \
1706   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1707   vdup.u32 dx4, uv_dx4[1];                                                     \
1708                                                                                \
1709   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1710   vdup.u32 dx8, uv_dx8[0];                                                     \
1711                                                                                \
1712   vadd.u32 u_block, u_block, dx8;                                              \
1713   vdup.u32 dx8, uv_dx8[1];                                                     \
1714                                                                                \
1715   vadd.u32 v_block, v_block, dx8;                                              \
1716   vmovn.u16 u_whole_8, u_whole;                                                \
1717                                                                                \
1718   vmovn.u16 v_whole_8, v_whole;                                                \
1719                                                                                \
1720   pld [ fb_ptr ];                                                              \
1721   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1722                                                                                \
1723   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1724   setup_blocks_texture_##swizzling();                                          \
1725                                                                                \
1726   beq 5f;                                                                      \
1727                                                                                \
1728  4:                                                                            \
1729   vshrn.u32 u_whole_low, u_block, #16;                                         \
1730                                                                                \
1731   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1732   vshrn.u32 v_whole_low, v_block, #16;                                         \
1733                                                                                \
1734   add block_ptr_b, block_ptr_b, #32;                                           \
1735   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1736                                                                                \
1737   vdup.u32 dx4, uv_dx4[0];                                                     \
1738   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1739   vdup.u32 dx4, uv_dx4[1];                                                     \
1740                                                                                \
1741   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1742   vdup.u32 dx8, uv_dx8[0];                                                     \
1743                                                                                \
1744   vadd.u32 u_block, u_block, dx8;                                              \
1745   vdup.u32 dx8, uv_dx8[1];                                                     \
1746                                                                                \
1747   vadd.u32 v_block, v_block, dx8;                                              \
1748   vmovn.u16 u_whole_8, u_whole;                                                \
1749                                                                                \
1750   add fb_ptr, fb_ptr, #16;                                                     \
1751   vmovn.u16 v_whole_8, v_whole;                                                \
1752                                                                                \
1753   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1754   pld [ fb_ptr ];                                                              \
1755                                                                                \
1756   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1757   subs span_num_blocks, span_num_blocks, #1;                                   \
1758                                                                                \
1759   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1760   setup_blocks_texture_##swizzling();                                          \
1761                                                                                \
1762   bne 4b;                                                                      \
1763                                                                                \
1764  5:                                                                            \
1765   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1766                                                                                \
1767   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1768   vdup.u8 draw_mask, right_mask;                                               \
1769                                                                                \
1770   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1771   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1772   vzip.u8 u_whole_8, v_whole_8;                                                \
1773                                                                                \
1774   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1775   add block_ptr_b, block_ptr_b, #32;                                           \
1776   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1777   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1778   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1779                                                                                \
1780  1:                                                                            \
1781   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1782   add span_edge_data, span_edge_data, #8;                                      \
1783   subs num_spans, num_spans, #1;                                               \
1784                                                                                \
1785   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1786   bne 0b;                                                                      \
1787                                                                                \
1788   ldmia sp!, { r4 - r11, pc };                                                 \
1789                                                                                \
1790  2:                                                                            \
1791   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1792   vpush { texture_mask };                                                      \
1793   vpush { uvrg_dx4 };                                                          \
1794                                                                                \
1795   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1796   bl flush_render_block_buffer;                                                \
1797   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1798                                                                                \
1799   vpop { uvrg_dx4 };                                                           \
1800   vpop { texture_mask };                                                       \
1801                                                                                \
1802   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1803   vmov.u8 fb_mask_ptrs, #0;                                                    \
1804                                                                                \
1805   mov num_blocks, span_num_blocks;                                             \
1806   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1807   bal 3b                                                                       \
1808
1809
1810 setup_blocks_unshaded_textured_builder(swizzled)
1811 setup_blocks_unshaded_textured_builder(unswizzled)
1812
1813
1814 .align 3
1815
1816 function(setup_blocks_unshaded_untextured_undithered_unswizzled_indirect)
1817   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1818   veor.u32 draw_mask, draw_mask, draw_mask
1819
1820   cmp num_spans, #0
1821   bxeq lr
1822
1823   stmdb sp!, { r4 - r11, r14 }
1824   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1825
1826   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1827
1828   ubfx color_r, color, #3, #5
1829   ubfx color_g, color, #11, #5
1830   ubfx color_b, color, #19, #5
1831
1832   orr color, color_r, color_b, lsl #10
1833   orr color, color, color_g, lsl #5
1834
1835   vdup.u16 colors, color
1836
1837   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1838   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1839
1840   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1841   add block_ptr_a, block_ptr_a, num_blocks, lsl #6
1842
1843  0:
1844   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1845   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1846
1847   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1848
1849   cmp span_num_blocks, #0
1850   beq 1f
1851
1852   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1853   add num_blocks, span_num_blocks, num_blocks
1854
1855   cmp num_blocks, #MAX_BLOCKS
1856   bgt 2f
1857
1858  3:
1859   add fb_ptr, fb_ptr, y, lsl #11
1860   and y, y, #0x3
1861
1862   add fb_ptr, fb_ptr, left_x, lsl #1
1863   mov c_32, #32
1864
1865   subs span_num_blocks, span_num_blocks, #1
1866
1867   add block_ptr_b, block_ptr_a, #16
1868   pld [ fb_ptr ]
1869
1870   vmov.u32 fb_mask_ptrs[1], fb_ptr
1871   beq 5f
1872
1873  4:
1874   vst1.u32 { draw_mask }, [ block_ptr_a, :128 ], c_32
1875   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1876   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1877
1878   add fb_ptr, fb_ptr, #16
1879   add block_ptr_b, block_ptr_b, #32
1880
1881   pld [ fb_ptr ]
1882
1883   vmov.u32 fb_mask_ptrs[1], fb_ptr
1884   subs span_num_blocks, span_num_blocks, #1
1885
1886   bne 4b
1887
1888  5:
1889   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1890
1891   vdup.u8 draw_mask_edge, right_mask
1892   vtst.u16 draw_mask_edge, draw_mask_edge, test_mask
1893
1894   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1895   vst1.u32 { draw_mask_edge }, [ block_ptr_a, :128 ], c_32
1896   add block_ptr_b, block_ptr_b, #32
1897   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1898
1899  1:
1900   add span_edge_data, span_edge_data, #8
1901   subs num_spans, num_spans, #1
1902
1903   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1904   bne 0b
1905
1906   ldmia sp!, { r4 - r11, pc }
1907                                                                            
1908  2:
1909   vpush { colors }
1910
1911   stmdb sp!, { r0 - r3, r12, r14 }
1912   bl flush_render_block_buffer
1913   ldmia sp!, { r0 - r3, r12, r14 }
1914
1915   vpop { colors }
1916
1917   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1918   veor.u32 draw_mask, draw_mask, draw_mask
1919
1920   mov num_blocks, span_num_blocks
1921   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1922   bal 3b
1923
1924
1925 #define mask_msb_scalar                                   r14
1926
1927 #define msb_mask                                          q15
1928
1929 #define pixels_low                                        d16
1930
1931 #define msb_mask_low                                      d30
1932 #define msb_mask_high                                     d31
1933
1934
1935 .align 3
1936
1937 function(setup_blocks_unshaded_untextured_undithered_unswizzled_direct)
1938   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1939
1940   cmp num_spans, #0
1941   bxeq lr
1942
1943   stmdb sp!, { r4 - r11, r14 }
1944
1945   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1946
1947   ubfx color_r, color, #3, #5
1948   ubfx color_g, color, #11, #5
1949
1950   ldrh mask_msb_scalar, [ psx_gpu, #psx_gpu_mask_msb_offset ]
1951   ubfx color_b, color, #19, #5
1952
1953   orr color, color_r, color_b, lsl #10
1954   orr color, color, color_g, lsl #5
1955   orr color, color, mask_msb_scalar
1956
1957   vdup.u16 colors, color
1958
1959   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1960
1961  0:
1962   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1963   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1964
1965   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1966
1967   cmp span_num_blocks, #0
1968   beq 1f
1969
1970   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1971
1972   add fb_ptr, fb_ptr, y, lsl #11
1973   subs span_num_blocks, span_num_blocks, #1
1974
1975   add fb_ptr, fb_ptr, left_x, lsl #1
1976   beq 3f
1977
1978  2:
1979   vst1.u32 { colors }, [ fb_ptr ]!
1980   subs span_num_blocks, span_num_blocks, #1
1981
1982   bne 2b
1983
1984  3:
1985   ldrb right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1986   eor right_mask, right_mask, #0xFF
1987
1988  4:
1989   strh color, [ fb_ptr ], #2
1990   movs right_mask, right_mask, lsr #1
1991   bne 4b
1992
1993  1:
1994   add span_edge_data, span_edge_data, #8
1995   subs num_spans, num_spans, #1
1996
1997   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1998   bne 0b
1999
2000   ldmia sp!, { r4 - r11, pc }
2001                                                                            
2002
2003
2004 #undef c_64
2005
2006 #define c_64                                              r7
2007 #define rg_dx_ptr                                         r2
2008
2009
2010 #undef r_block
2011 #undef g_block
2012 #undef b_block
2013 #undef r_whole
2014 #undef g_whole
2015 #undef b_whole
2016 #undef r_whole_low
2017 #undef r_whole_high
2018 #undef g_whole_low
2019 #undef g_whole_high
2020 #undef b_whole_low
2021 #undef b_whole_high
2022 #undef r_whole_8
2023 #undef g_whole_8
2024 #undef b_whole_8
2025 #undef dither_offsets
2026 #undef rg_dx4
2027 #undef rg_dx8
2028 #undef dx4
2029 #undef dx8
2030 #undef v_left_x
2031 #undef uvrg
2032 #undef block_span
2033 #undef rg
2034 #undef draw_mask
2035 #undef test_mask
2036
2037 #define r_block                                           q0
2038 #define g_block                                           q1
2039 #define b_block                                           q2
2040
2041 #define r_whole                                           q3
2042 #define g_whole                                           q4
2043 #define b_whole                                           q5
2044
2045 #define r_whole_low                                       d6
2046 #define r_whole_high                                      d7
2047 #define g_whole_low                                       d8
2048 #define g_whole_high                                      d9
2049 #define b_whole_low                                       d10
2050 #define b_whole_high                                      d11
2051
2052 #define gb_whole_8                                        q6
2053
2054 #define g_whole_8                                         d12
2055 #define b_whole_8                                         d13
2056
2057 #define r_whole_8                                         d14
2058
2059 #define pixels                                            q8
2060
2061 #define rg_dx4                                            d18
2062 #define rg_dx8                                            d19
2063
2064 #define dx4                                               q10
2065 #define dx8                                               q10
2066
2067 #define v_left_x                                          d6
2068 #define uvrg                                              q4
2069 #define block_span                                        q5
2070
2071 #define rg                                                d9
2072
2073 #define d64_1                                             d22
2074 #define d64_128                                           d23
2075
2076 #define d128_4                                            q12
2077 #define d128_0x7                                          q13
2078
2079 #define d64_4                                             d24
2080
2081 #define dither_offsets                                    q14
2082 #define draw_mask                                         q15
2083
2084 #define dither_offsets_low                                d28
2085
2086 #define rg_dx                                             d0
2087 #define test_mask                                         q10
2088
2089
2090 #define setup_blocks_shaded_untextured_dither_a_dithered()                     \
2091   vqadd.u8 r_whole_8, r_whole_8, dither_offsets_low;                           \
2092   vqadd.u8 gb_whole_8, gb_whole_8, dither_offsets;                             \
2093
2094 #define setup_blocks_shaded_untextured_dither_b_dithered()                     \
2095   vqsub.u8 r_whole_8, r_whole_8, d64_4;                                        \
2096   vqsub.u8 gb_whole_8, gb_whole_8, d128_4                                      \
2097
2098 #define setup_blocks_shaded_untextured_dither_a_undithered()                   \
2099
2100 #define setup_blocks_shaded_untextured_dither_b_undithered()                   \
2101
2102
2103 #define setup_blocks_shaded_untextured_indirect_builder(dithering)             \
2104 .align 3;                                                                      \
2105                                                                                \
2106 function(setup_blocks_shaded_untextured_##dithering##_unswizzled_indirect)     \
2107   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
2108   add rg_dx_ptr, psx_gpu, #(psx_gpu_uvrg_dx_offset + 8);                       \
2109                                                                                \
2110   vld1.u32 { rg_dx }, [ rg_dx_ptr, :64 ];                                      \
2111                                                                                \
2112   cmp num_spans, #0;                                                           \
2113   bxeq lr;                                                                     \
2114                                                                                \
2115   stmdb sp!, { r4 - r11, r14 };                                                \
2116   vshl.u32 rg_dx4, rg_dx, #2;                                                  \
2117                                                                                \
2118   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
2119   vshl.u32 rg_dx8, rg_dx, #3;                                                  \
2120                                                                                \
2121   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
2122                                                                                \
2123   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
2124   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
2125                                                                                \
2126   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
2127   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
2128                                                                                \
2129   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
2130   vmov.u8 d64_1, #1;                                                           \
2131                                                                                \
2132   vmov.u8 d128_4, #4;                                                          \
2133   vmov.u8 d64_128, #128;                                                       \
2134                                                                                \
2135   vmov.u8 d128_0x7, #0x7;                                                      \
2136                                                                                \
2137  0:                                                                            \
2138   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
2139   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
2140                                                                                \
2141   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
2142   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
2143                                                                                \
2144   cmp span_num_blocks, #0;                                                     \
2145   beq 1f;                                                                      \
2146                                                                                \
2147   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
2148   add num_blocks, span_num_blocks, num_blocks;                                 \
2149                                                                                \
2150   cmp num_blocks, #MAX_BLOCKS;                                                 \
2151   bgt 2f;                                                                      \
2152                                                                                \
2153  3:                                                                            \
2154   ldr b, [ span_b_offset ];                                                    \
2155   add fb_ptr, fb_ptr, y, lsl #11;                                              \
2156                                                                                \
2157   vdup.u32 v_left_x, left_x;                                                   \
2158   and y, y, #0x3;                                                              \
2159                                                                                \
2160   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
2161   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
2162                                                                                \
2163   mla b, b_dx, left_x, b;                                                      \
2164   and dither_shift, left_x, #0x03;                                             \
2165                                                                                \
2166   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
2167   vshr.u32 rg_dx, rg_dx4, #2;                                                  \
2168                                                                                \
2169   mov dither_shift, dither_shift, lsl #3;                                      \
2170   vmla.u32 rg, rg_dx, v_left_x;                                                \
2171                                                                                \
2172   mov c_64, #64;                                                               \
2173   subs span_num_blocks, span_num_blocks, #1;                                   \
2174                                                                                \
2175   mov dither_row, dither_row, ror dither_shift;                                \
2176   mov b_dx4, b_dx, lsl #2;                                                     \
2177                                                                                \
2178   vdup.u32 dither_offsets, dither_row;                                         \
2179   add block_span_ptr, psx_gpu, #psx_gpu_r_block_span_offset;                   \
2180                                                                                \
2181   vdup.u32 b_block, b;                                                         \
2182   vadd.u8 dither_offsets, dither_offsets, d128_4;                              \
2183                                                                                \
2184   mov b_dx8, b_dx, lsl #3;                                                     \
2185   vdup.u32 r_block, rg[0];                                                     \
2186   vdup.u32 g_block, rg[1];                                                     \
2187                                                                                \
2188   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2189                                                                                \
2190   vadd.u32 r_block, r_block, block_span;                                       \
2191   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2192                                                                                \
2193   vadd.u32 g_block, g_block, block_span;                                       \
2194   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
2195                                                                                \
2196   vadd.u32 b_block, b_block, block_span;                                       \
2197   add block_ptr_b, block_ptr_a, #16;                                           \
2198                                                                                \
2199   vshrn.u32 r_whole_low, r_block, #16;                                         \
2200   vshrn.u32 g_whole_low, g_block, #16;                                         \
2201   vshrn.u32 b_whole_low, b_block, #16;                                         \
2202   vdup.u32 dx4, rg_dx4[0];                                                     \
2203                                                                                \
2204   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2205   vdup.u32 dx4, rg_dx4[1];                                                     \
2206                                                                                \
2207   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2208   vdup.u32 dx4, b_dx4;                                                         \
2209                                                                                \
2210   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2211   vdup.u32 dx8, rg_dx8[0];                                                     \
2212                                                                                \
2213   vadd.u32 r_block, r_block, dx8;                                              \
2214   vdup.u32 dx8, rg_dx8[1];                                                     \
2215                                                                                \
2216   vadd.u32 g_block, g_block, dx8;                                              \
2217   vdup.u32 dx8, b_dx8;                                                         \
2218                                                                                \
2219   vadd.u32 b_block, b_block, dx8;                                              \
2220                                                                                \
2221   vmovn.u16 r_whole_8, r_whole;                                                \
2222   vmovn.u16 g_whole_8, g_whole;                                                \
2223   vmovn.u16 b_whole_8, b_whole;                                                \
2224                                                                                \
2225   beq 5f;                                                                      \
2226   veor.u32 draw_mask, draw_mask, draw_mask;                                    \
2227                                                                                \
2228  4:                                                                            \
2229   setup_blocks_shaded_untextured_dither_a_##dithering();                       \
2230   vshrn.u32 r_whole_low, r_block, #16;                                         \
2231                                                                                \
2232   setup_blocks_shaded_untextured_dither_b_##dithering();                       \
2233   vshrn.u32 g_whole_low, g_block, #16;                                         \
2234                                                                                \