3551b5960157e7b0676e87a9f8e7ca23268a6713
[pcsx_rearmed.git] / plugins / gpu_neon / psx_gpu / psx_gpu_arm_neon.S
1 /*
2  * Copyright (C) 2011 Gilead Kutnick "Exophase" <exophase@gmail.com>
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of
7  * the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  */
14
15 #define MAX_SPANS                                         512
16 #define MAX_BLOCKS                                        64
17 #define MAX_BLOCKS_PER_ROW                                128
18
19 #define psx_gpu_test_mask_offset                          0
20 #define psx_gpu_uvrg_offset                               16
21 #define psx_gpu_uvrg_dx_offset                            32
22 #define psx_gpu_uvrg_dy_offset                            48
23 #define psx_gpu_u_block_span_offset                       64
24 #define psx_gpu_v_block_span_offset                       80
25 #define psx_gpu_r_block_span_offset                       96
26 #define psx_gpu_g_block_span_offset                       112
27 #define psx_gpu_b_block_span_offset                       128
28
29 #define psx_gpu_b_dx_offset                               132
30
31 #define psx_gpu_b_offset                                  144
32 #define psx_gpu_b_dy_offset                               148
33 #define psx_gpu_triangle_area_offset                      152
34 #define psx_gpu_texture_window_settings_offset            156
35 #define psx_gpu_current_texture_mask_offset               160
36 #define psx_gpu_viewport_mask_offset                      164
37 #define psx_gpu_dirty_textures_4bpp_mask_offset           168
38 #define psx_gpu_dirty_textures_8bpp_mask_offset           172
39 #define psx_gpu_dirty_textures_8bpp_alternate_mask_offset 176
40 #define psx_gpu_triangle_color_offset                     180
41 #define psx_gpu_primitive_color_offset                    184
42 #define psx_gpu_dither_table_offset                       188
43 #define psx_gpu_render_block_handler_offset               204
44 #define psx_gpu_texture_page_ptr_offset                   208
45 #define psx_gpu_clut_ptr_offset                           212
46 #define psx_gpu_vram_ptr_offset                           216
47
48 #define psx_gpu_render_state_base_offset                  220
49 #define psx_gpu_render_state_offset                       222
50 #define psx_gpu_num_spans_offset                          224
51 #define psx_gpu_num_blocks_offset                         226
52 #define psx_gpu_offset_x_offset                           228
53 #define psx_gpu_offset_y_offset                           230
54 #define psx_gpu_clut_settings_offset                      232
55 #define psx_gpu_texture_settings_offset                   234
56 #define psx_gpu_viewport_start_x_offset                   236
57 #define psx_gpu_viewport_start_y_offset                   238
58 #define psx_gpu_viewport_end_x_offset                     240
59 #define psx_gpu_viewport_end_y_offset                     242
60 #define psx_gpu_mask_msb_offset                           244
61                                                           
62 #define psx_gpu_triangle_winding_offset                   246
63 #define psx_gpu_display_area_draw_enable_offset           247
64 #define psx_gpu_current_texture_page_offset               248
65 #define psx_gpu_last_8bpp_texture_page_offset             249
66 #define psx_gpu_texture_mask_width_offset                 250
67 #define psx_gpu_texture_mask_height_offset                251
68 #define psx_gpu_texture_window_x_offset                   252
69 #define psx_gpu_texture_window_y_offset                   253
70 #define psx_gpu_primitive_type_offset                     254
71
72 #define psx_gpu_reserved_a_offset                         255
73
74 #define psx_gpu_blocks_offset                             0x0100
75 #define psx_gpu_span_uvrg_offset_offset                   0x2100
76 #define psx_gpu_span_edge_data_offset                     0x4100
77 #define psx_gpu_span_b_offset_offset                      0x5100
78
79 #define edge_data_left_x_offset                           0
80 #define edge_data_num_blocks_offset                       2
81 #define edge_data_right_mask_offset                       4
82 #define edge_data_y_offset                                6
83
84
85 #define psx_gpu                                           r0
86 #define v_a                                               r1
87 #define v_b                                               r2
88 #define v_c                                               r3
89
90 #define x0                                                r4
91 #define x1                                                r5
92 #define x2                                                r6
93 #define x0_x1                                             r5
94 #define x1_x2                                             r6
95 #define y0                                                r7
96 #define y1                                                r8
97 #define y2                                                r9
98 #define y0_y1                                             r7
99 #define y1_y2                                             r8
100 #define b0                                                r9
101 #define b1                                                r10
102 #define b2                                                r11
103 #define b0_b1                                             r10
104 #define b1_b2                                             r11
105
106
107 #define area_r_s                                          r5
108
109 #define g_bx0                                             r2
110 #define g_bx                                              r3
111 #define g_bx2                                             r4
112 #define g_bx3                                             r5
113 #define b_base                                            r6
114 #define g_by                                              r8
115
116 #define gs_bx                                             r7
117 #define gs_by                                             r10
118
119 #define ga_bx                                             g_bx
120 #define ga_by                                             g_by
121
122 #define gw_bx_h                                           g_bx
123 #define gw_by_h                                           g_by
124
125 #define gw_bx_l                                           r11
126 #define gw_by_l                                           gw_bx_l
127
128 #define store_a                                           r0
129 #define store_b                                           r1
130 #define store_inc                                         r5
131
132
133 #define v0                                                q0
134 #define uvrgb0                                            d0
135 #define x0_y0                                             d1
136
137 #define v1                                                q1
138 #define uvrgb1                                            d2
139 #define x1_y1                                             d3
140
141 #define v2                                                q2
142 #define uvrgb2                                            d4
143 #define x2_y2                                             d5
144
145 #define x0_ab                                             q3
146 #define uvrg_xxxx0                                        q3
147 #define uvrg0                                             d6
148 #define xxxx0                                             d7
149
150 #define x1_ab                                             q4
151 #define uvrg_xxxx1                                        q4
152 #define uvrg1                                             d8
153 #define xxxx1                                             d9
154
155 #define x2_ab                                             q5
156 #define uvrg_xxxx2                                        q5
157 #define uvrg2                                             d10
158 #define xxxx2                                             d11
159
160 #define y0_ab                                             q6
161 #define yyyy_uvrg0                                        q6
162 #define yyyy0                                             d12
163 #define uvrg0b                                            d13
164
165 #define y1_ab                                             q7
166 #define yyyy_uvrg1                                        q7
167 #define yyyy1                                             d14
168 #define uvrg1b                                            d15
169
170 #define y2_ab                                             q8
171 #define yyyy_uvrg2                                        q8
172 #define yyyy2                                             d16
173 #define uvrg2b                                            d17
174
175 #define d0_ab                                             q9
176 #define d0_a                                              d18
177 #define d0_b                                              d19
178
179 #define d1_ab                                             q10
180 #define d1_a                                              d20
181 #define d1_b                                              d21
182
183 #define d2_ab                                             q11
184 #define d2_a                                              d22
185 #define d2_b                                              d23
186
187 #define d3_ab                                             q12
188 #define d3_a                                              d24
189 #define d3_b                                              d25
190
191 #define ga_uvrg_x                                         q1
192 #define ga_uvrg_y                                         q4
193
194 #define dx                                                x0_x1
195 #define dy                                                y0_y1
196 #define db                                                b0_b1
197
198 #define uvrg_base                                         q11
199
200 #define gs_uvrg_x                                         q5
201 #define gs_uvrg_y                                         q6
202
203 #define g_uvrg_x                                          q1
204 #define ga_uv_x                                           d2
205 #define g_uv_x                                            d2
206 #define ga_rg_x                                           d3
207 #define g_rg_x                                            d3
208
209 #define g_uvrg_y                                          q4
210 #define ga_uv_y                                           d8
211 #define g_uv_y                                            d8
212 #define ga_rg_y                                           d9
213 #define g_rg_y                                            d9
214
215 #define gw_uv_x                                           q1
216 #define gw_rg_x                                           q2
217 #define gw_uv_y                                           q4
218 #define gw_rg_y                                           q3
219
220 #define w_mask                                            q9
221 #define w_mask_l                                          d18
222
223 #define r_shift                                           q10
224
225 #define uvrg_dx0                                          q0
226 #define uvrg_dx0l                                         d0
227 #define uvrg_dx0h                                         d1
228
229 #define uvrg_dx1                                          q1
230 #define uvrg_dx1l                                         d2
231 #define uvrg_dx1h                                         d3
232
233 #define uvrg_dx2                                          q2
234 #define uvrg_dx2l                                         d4
235 #define uvrg_dx2h                                         d5
236
237 #define uvrg_dx3                                          q3
238 #define uvrg_dx3l                                         d6
239 #define uvrg_dx3h                                         d7
240
241
242 .align 4
243
244 #define function(name)                                                         \
245   .global name;                                                                \
246   name:                                                                        \
247
248 @ r0: psx_gpu
249 @ r1: v_a
250 @ r2: v_b
251 @ r3: v_c
252
253 function(compute_all_gradients)
254   // First compute the triangle area reciprocal and shift. The division will
255   // happen concurrently with much of the work which follows.
256   @ r12 = psx_gpu->triangle_area
257   ldr r12, [ psx_gpu, #psx_gpu_triangle_area_offset ]
258   stmdb sp!, { r4 - r11, lr }
259
260   @ load exponent of 62 into upper half of double
261   movw r4, #0
262   clz r14, r12                       @ r14 = shift
263
264   movt r4, #((62 + 1023) << 4)
265   mov r12, r12, lsl r14              @ r12 = triangle_area_normalized
266
267   @ load area normalized into lower half of double
268   mov r5, r12, lsr #10
269   vmov.f64 d30, r5, r4               @ d30 = (1 << 62) + ta_n
270
271   movt r4, #((1022 + 31) << 4)
272   mov r5, r12, lsl #20
273
274   add r4, r4, r12, lsr #11
275   vmov.f64 d31, r5, r4
276
277   vdiv.f64 d30, d30, d31             @ d30 = ((1 << 62) + ta_n) / ta_n
278
279   // ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0)) =
280   // ( d0       *  d1      ) - ( d2       *  d3      ) =
281   // ( m0                  ) - ( m1                  ) = gradient
282
283   // This is split to do 12 elements at a time over three sets: a, b, and c.
284   // Technically we only need to do 10 elements (uvrgb_x and uvrgb_y), so
285   // two of the slots are unused.
286
287   // Inputs are all 16-bit signed. The m0/m1 results are 32-bit signed, as
288   // is g.
289
290   // First type is:  uvrg bxxx xxxx 
291   // Second type is: yyyy ybyy uvrg 
292   // Since x_a and y_c are the same the same variable is used for both. 
293
294   vld1.u32 { v0 }, [ v_a, : 128 ]    @ v0 = { uvrg0, b0, x0, y0 }
295   ldrsh x0, [ v_a, #8 ]              @ load x0
296
297   vld1.u32 { v1 }, [ v_b, : 128 ]    @ v1 = { uvrg1, b1, x1, y1}
298   ldrh x1, [ v_b, #8 ]               @ load x1
299
300   vld1.u32 { v2 }, [ v_c, : 128 ]    @ v2 = { uvrg2, b2, x2, y2 }
301   ldrh x2, [ v_c, #8 ]               @ load x2
302
303   vmovl.u8 uvrg_xxxx0, uvrgb0        @ uvrg_xxxx0 = { uv0, rg0, b0-, -- }
304   ldrh y0, [ v_a, #10 ]              @ load y0
305
306   vmovl.u8 uvrg_xxxx1, uvrgb1        @ uvrg_xxxx1 = { uv1, rg1, b1-, -- }
307   ldrh y1, [ v_b, #10 ]              @ load y1
308
309   vmovl.u8 uvrg_xxxx2, uvrgb2        @ uvrg_xxxx2 = { uv2, rg2, b2-, -- }
310   ldrh y2, [ v_c, #10 ]              @ load y2
311
312   vmov.u8 uvrg0b, uvrg0              @ uvrg0b = { uv0, rg0 }
313   vdup.u16 xxxx0, x0_y0[0]           @ xxxx0 = { xx0, xx0 }
314
315   orr x1_x2, x1, x2, lsl #16         @ x1_x2 = { x1, x2 }
316   pkhbt x0_x1, x0, x1, lsl #16       @ x0_x1 = { x0, x1 }
317
318   vmov.u8 uvrg1b, uvrg1              @ uvrg1b = { uv1, rg1 }
319   vdup.u16 xxxx1, x1_y1[0]           @ xxxx1 = { xx1, xx1 }
320
321   vmov.u8 uvrg2b, uvrg2              @ uvrg2b = { uv2, rg2 }
322   vdup.u16 xxxx2, x2_y2[0]           @ xxxx2 = { xx2, xx2 }
323
324   ldrb b2, [ v_c, #4 ]               @ load b2
325   orr y0_y1, y0, y1, lsl #16         @ y0_y1 = { y0, y1 }
326
327   ldrb b1, [ v_b, #4 ]               @ load b1
328   orr y1_y2, y1, y2, lsl #16         @ y1_y2 = { y1, y2 }
329
330   vdup.u16 yyyy0, x0_y0[1]           @ yyyy0 = { yy0, yy0 }
331   vsub.s16 d0_ab, x1_ab, x0_ab
332
333   ldrb b0, [ v_a, #4 ]               @ load b0
334   orr b1_b2, b1, b2, lsl #16         @ b1_b2 = { b1, b2 }
335
336   vdup.u16 yyyy1, x1_y1[1]           @ yyyy1 = { yy1, yy1 }
337   vsub.s16 d2_ab, x2_ab, x1_ab
338
339   vdup.u16 yyyy2, x2_y2[1]           @ yyyy2 = { yy2, yy2 }
340   vsub.s16 d1_ab, y2_ab, y1_ab
341
342   orr b0_b1, b0, b1, lsl #16         @ b1_b2 = { b1, b2 }
343   ssub16 dx, x1_x2, x0_x1            @ dx = { x1 - x0, x2 - x1 }
344
345   ssub16 dy, y1_y2, y0_y1            @ dy = { y1 - y0, y2 - y1 }
346   ssub16 db, b1_b2, b0_b1            @ db = { b1 - b0, b2 - b1 }
347
348   vsub.s16 d3_ab, y1_ab, y0_ab
349   smusdx ga_by, dx, db               @ ga_by = ((x1 - x0) * (b2 - b1)) -
350                                      @         ((x2 - X1) * (b1 - b0)) 
351   vmull.s16 ga_uvrg_x, d0_a, d1_a
352   smusdx ga_bx, db, dy               @ ga_bx = ((b1 - b0) * (y2 - y1)) -
353                                      @         ((b2 - b1) * (y1 - y0))
354   vmlsl.s16 ga_uvrg_x, d2_a, d3_a
355   movs gs_bx, ga_bx, asr #31
356
357   vmull.s16 ga_uvrg_y, d0_b, d1_b
358   rsbmi ga_bx, ga_bx, #0
359
360   vmlsl.s16 ga_uvrg_y, d2_b, d3_b
361   movs gs_by, ga_by, asr #31
362
363   vshr.u64 d0, d30, #22
364   mov b_base, b0, lsl #16
365
366   rsbmi ga_by, ga_by, #0
367   vclt.s32 gs_uvrg_x, ga_uvrg_x, #0  @ gs_uvrg_x = ga_uvrg_x < 0
368
369   @ r12 = psx_gpu->triangle_winding_offset
370   ldrb r12, [ psx_gpu, #psx_gpu_triangle_winding_offset ]
371   vclt.s32 gs_uvrg_y, ga_uvrg_y, #0  @ gs_uvrg_y = ga_uvrg_y < 0
372
373   add b_base, b_base, #0x8000
374   rsb r12, r12, #0                   @ r12 = -(triangle->winding)
375
376   vdup.u32 w_mask, r12               @ w_mask = { -w, -w, -w, -w }
377   sub r14, r14, #(62 - 12)           @ r14 = shift - (62 - FIXED_BITS)
378
379   vshll.u16 uvrg_base, uvrg0, #16    @ uvrg_base = uvrg0 << 16
380   vdup.u32 r_shift, r14              @ r_shift = { shift, shift, shift, shift }
381
382   vorr.u32 uvrg_base, #0x8000
383   vabs.s32 ga_uvrg_x, ga_uvrg_x      @ ga_uvrg_x = abs(ga_uvrg_x)
384
385   vmov area_r_s, s0                  @ area_r_s = triangle_reciprocal
386   vabs.s32 ga_uvrg_y, ga_uvrg_y      @ ga_uvrg_y = abs(ga_uvrg_y)
387
388   vmull.u32 gw_rg_x, ga_rg_x, d0[0]
389   vmull.u32 gw_uv_x, ga_uv_x, d0[0]
390   vmull.u32 gw_rg_y, ga_rg_y, d0[0]
391   vmull.u32 gw_uv_y, ga_uv_y, d0[0]
392
393   vshl.u64 gw_rg_x, gw_rg_x, r_shift
394   vshl.u64 gw_uv_x, gw_uv_x, r_shift
395   vshl.u64 gw_rg_y, gw_rg_y, r_shift
396   vshl.u64 gw_uv_y, gw_uv_y, r_shift
397
398   veor.u32 gs_uvrg_x, gs_uvrg_x, w_mask
399   vmovn.u64 g_uv_x, gw_uv_x
400
401   veor.u32 gs_uvrg_y, gs_uvrg_y, w_mask
402   vmovn.u64 g_rg_x, gw_rg_x
403
404   veor.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
405   vmovn.u64 g_uv_y, gw_uv_y
406
407   vsub.u32 g_uvrg_x, g_uvrg_x, gs_uvrg_x
408   vmovn.u64 g_rg_y, gw_rg_y
409
410   veor.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
411   mov ga_bx, ga_bx, lsl #13
412
413   vsub.u32 g_uvrg_y, g_uvrg_y, gs_uvrg_y
414   mov ga_by, ga_by, lsl #13
415
416   vdup.u32 x0_y0, x0
417   umull gw_bx_l, gw_bx_h, ga_bx, area_r_s
418
419   vshl.u32 g_uvrg_x, g_uvrg_x, #4
420   vshl.u32 g_uvrg_y, g_uvrg_y, #4
421
422   umull gw_by_l, gw_by_h, ga_by, area_r_s
423   vmls.s32 uvrg_base, ga_uvrg_x, x0_y0[0]
424
425   eor gs_bx, gs_bx, r12
426   vadd.u32 uvrg_dx2, uvrg_dx1, uvrg_dx1
427
428   veor.u32 uvrg_dx0, uvrg_dx0, uvrg_dx0
429   eor gs_by, gs_by, r12
430
431   rsb r11, r14, #0                   @ r11 = negative shift for scalar lsr
432   add store_a, psx_gpu, #psx_gpu_uvrg_offset
433
434   sub r11, r11, #(32 - 13)
435
436   add store_b, store_a, #16
437   mov store_inc, #32
438
439   vadd.u32 uvrg_dx3, uvrg_dx2, uvrg_dx1
440   vst1.u32 { uvrg_base }, [ store_a, : 128 ], store_inc
441
442   vst1.u32 { uvrg_dx1 }, [ store_b, : 128 ], store_inc
443   mov g_bx, gw_bx_h, lsr r11
444
445   vst1.u32 { g_uvrg_y }, [ store_a, : 128 ], store_inc
446   mov g_by, gw_by_h, lsr r11
447
448   vst4.u32 { uvrg_dx0l, uvrg_dx1l, uvrg_dx2l, uvrg_dx3l },                     \
449    [ store_b, : 128 ], store_inc
450   eor g_bx, g_bx, gs_bx
451
452   vst4.u32 { uvrg_dx0h, uvrg_dx1h, uvrg_dx2h, uvrg_dx3h },                     \
453    [ store_b, : 128 ], store_inc
454   sub g_bx, g_bx, gs_bx
455
456   lsl g_bx, g_bx, #4  
457   eor g_by, g_by, gs_by
458
459   mls b_base, g_bx, x0, b_base
460   sub g_by, g_by, gs_by
461
462   lsl g_by, g_by, #4
463   mov g_bx0, #0
464
465   add g_bx2, g_bx, g_bx
466   add g_bx3, g_bx, g_bx2
467
468   stmia store_b, { g_bx0, g_bx, g_bx2, g_bx3, b_base, g_by }
469
470   ldmia sp!, { r4 - r11, pc }
471
472
473 #define psx_gpu                                  r0
474 #define v_a                                      r1
475 #define v_b                                      r2
476 #define v_c                                      r3
477
478 #define temp                                     r14
479
480 #define x_a                                      r4
481 #define x_b                                      r5
482 #define x_c                                      r6
483 #define y_a                                      r1
484 #define y_b                                      r2
485 #define y_c                                      r3
486
487 #define height_minor_a                           r7
488 #define height_minor_b                           r8
489 #define height_major                             r9
490 #define height                                   r9
491
492 #define reciprocal_table_ptr                     r10
493
494 #define edge_alt_low                             r4
495 #define edge_alt_high                            r5
496 #define edge_dx_dy_alt                           r6
497 #define edge_shift_alt                           r10
498
499 #define edge_dx_dy_alt_low                       r4
500 #define edge_dx_dy_alt_high                      r5
501
502 #define span_edge_data                           r4
503 #define span_uvrg_offset                         r5
504 #define span_b_offset                            r6
505
506 #define clip                                     r14
507
508 #define b                                        r11
509 #define b_dy                                     r12
510
511
512 #define alternate_x                              q0
513 #define alternate_dx_dy                          q1
514 #define alternate_x_32                           q2
515
516 #define alternate_x_low                          d0
517 #define alternate_x_high                         d1
518 #define alternate_dx_dy_low                      d2
519 #define alternate_dx_dy_high                     d3
520 #define alternate_x_32_low                       d4
521 #define alternate_x_32_high                      d5
522
523 #define left_x                                   q3
524 #define right_x                                  q4
525 #define left_dx_dy                               q5
526 #define right_dx_dy                              q6
527 #define left_edge                                q7
528 #define right_edge                               q8
529
530 #define left_x_low                               d6
531 #define left_x_high                              d7
532 #define right_x_low                              d8
533 #define right_x_high                             d9
534 #define left_dx_dy_low                           d10
535 #define left_dx_dy_high                          d11
536 #define right_dx_dy_low                          d12
537 #define right_dx_dy_high                         d13
538 #define left_edge_low                            d14
539 #define left_edge_high                           d15
540 #define right_edge_low                           d16
541 #define right_edge_high                          d17
542
543 #define y_mid_point                              d18
544 #define c_0x0004                                 d19
545
546 #define left_right_x_16                          q11
547 #define span_shifts_y                            q12
548 #define c_0x0001                                 q13
549
550 #define span_shifts                              d24
551 #define y_x4                                     d25
552 #define c_0xFFFE                                 d26
553 #define c_0x0007                                 d27
554
555 #define left_right_x_16_low                      d22
556 #define left_right_x_16_high                     d23
557
558 #define uvrg                                     q14
559 #define uvrg_dy                                  q15
560
561 #define alternate_x_16                           d4
562
563 #define v_clip                                   q3
564 #define v_clip_low                               d6
565
566 #define right_x_32                               q10
567 #define left_x_32                                q11
568 #define alternate_select                         d24
569
570 #define right_x_32_low                           d20
571 #define right_x_32_high                          d21
572 #define left_x_32_low                            d22
573 #define left_x_32_high                           d23
574
575 #define edges_xy                                 q0
576 #define edges_dx_dy                              d2
577 #define edge_shifts                              d3
578 #define edge_shifts_64                           q2
579
580 #define edges_xy_left                            d0
581 #define edges_xy_right                           d1
582
583 #define height_reciprocals                       d6
584 #define heights                                  d7
585
586 #define widths                                   d8
587 #define c_0x01                                   d9
588 #define x_starts                                 d10
589 #define x_ends                                   d11
590
591 #define heights_b                                d12
592 #define edges_dx_dy_64                           q10
593
594 #define edges_dx_dy_64_left                      d20
595 #define edges_dx_dy_64_right                     d21
596
597
598 #define setup_spans_prologue()                                                 \
599   stmdb sp!, { r4 - r11, lr };                                                 \
600                                                                                \
601   ldrsh x_a, [ v_a, #8 ];                                                      \
602   ldrsh x_b, [ v_b, #8 ];                                                      \
603   ldrsh x_c, [ v_c, #8 ];                                                      \
604   ldrsh y_a, [ v_a, #10 ];                                                     \
605   ldrsh y_b, [ v_b, #10 ];                                                     \
606   ldrsh y_c, [ v_c, #10 ];                                                     \
607                                                                                \
608   add temp, psx_gpu, #psx_gpu_uvrg_offset;                                     \
609   vld1.32 { uvrg }, [ temp ];                                                  \
610   add temp, psx_gpu, #psx_gpu_uvrg_dy_offset;                                  \
611   vld1.32 { uvrg_dy }, [ temp ];                                               \
612   movw reciprocal_table_ptr, :lower16:reciprocal_table;                        \
613   movt reciprocal_table_ptr, :upper16:reciprocal_table;                        \
614                                                                                \
615   vmov.u32 c_0x01, #0x01                                                       \
616
617 #define setup_spans_load_b()                                                   \
618   ldr b, [ psx_gpu, #psx_gpu_b_offset ];                                       \
619   ldr b_dy, [ psx_gpu, #psx_gpu_b_dy_offset ]                                  \
620
621 #define setup_spans_prologue_b()                                               \
622   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
623   add temp, psx_gpu, #psx_gpu_viewport_start_x_offset;                         \
624                                                                                \
625   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
626   vmov.u16 c_0x0004, #0x0004;                                                  \
627                                                                                \
628   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
629   vmov.u16 c_0x0001, #0x0001;                                                  \
630                                                                                \
631   vld1.u16 { left_edge_low[], left_edge_high[] }, [ temp ];                    \
632   add temp, psx_gpu, #psx_gpu_viewport_end_x_offset;                           \
633                                                                                \
634   vld1.u16 { right_edge_low[], right_edge_high[] }, [ temp ];                  \
635   vadd.u16 right_edge, right_edge, c_0x0001;                                   \
636                                                                                \
637   vmov.u16 c_0x0007, #0x0007;                                                  \
638   vmvn.u16 c_0xFFFE, #0x0001                                                   \
639
640
641 #define compute_edge_delta_x2()                                                \
642   ldr temp, [ reciprocal_table_ptr, height, lsl #2 ];                          \
643                                                                                \
644   vdup.u32 heights, height;                                                    \
645   vsub.u32 widths, x_ends, x_starts;                                           \
646                                                                                \
647   vdup.u32 edge_shifts, temp;                                                  \
648   vsub.u32 heights_b, heights, c_0x01;                                         \
649   vshr.u32 height_reciprocals, edge_shifts, #12;                               \
650                                                                                \
651   vmla.s32 heights_b, x_starts, heights;                                       \
652   vbic.u16 edge_shifts, #0xE0;                                                 \
653   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
654   vmull.s32 edges_xy, heights_b, height_reciprocals                            \
655
656 #define width_alt                 r6
657 #define height_reciprocal_alt     r11
658 #define height_b_alt              r12
659
660 #define compute_edge_delta_x3(start_c, height_a, height_b)                     \
661   vmov.u32 heights, height_a, height_b;                                        \
662   ldr temp, [ reciprocal_table_ptr, height_a, lsl #2 ];                        \
663   vmov.u32 edge_shifts[0], temp;                                               \
664   ldr temp, [ reciprocal_table_ptr, height_b, lsl #2 ];                        \
665   vmov.u32 edge_shifts[1], temp;                                               \
666   ldr edge_shift_alt, [ reciprocal_table_ptr, height_minor_b, lsl #2 ];        \
667                                                                                \
668   vsub.u32 widths, x_ends, x_starts;                                           \
669   sub width_alt, x_c, start_c;                                                 \
670                                                                                \
671   vsub.u32 heights_b, heights, c_0x01;                                         \
672   sub height_b_alt, height_minor_b, #1;                                        \
673                                                                                \
674   vshr.u32 height_reciprocals, edge_shifts, #12;                               \
675   lsr height_reciprocal_alt, edge_shift_alt, #12;                              \
676                                                                                \
677   vmla.s32 heights_b, x_starts, heights;                                       \
678   mla height_b_alt, height_minor_b, start_c, height_b_alt;                     \
679                                                                                \
680   vbic.u16 edge_shifts, #0xE0;                                                 \
681   and edge_shift_alt, edge_shift_alt, #0x1F;                                   \
682                                                                                \
683   vmul.s32 edges_dx_dy, widths, height_reciprocals;                            \
684   mul edge_dx_dy_alt, width_alt, height_reciprocal_alt;                        \
685                                                                                \
686   vmull.s32 edges_xy, heights_b, height_reciprocals;                           \
687   smull edge_alt_low, edge_alt_high, height_b_alt, height_reciprocal_alt       \
688
689
690 #define setup_spans_adjust_y_up()                                              \
691   vsub.u32 y_x4, y_x4, c_0x0004                                                \
692
693 #define setup_spans_adjust_y_down()                                            \
694   vadd.u32 y_x4, y_x4, c_0x0004                                                \
695
696 #define setup_spans_adjust_interpolants_up()                                   \
697   vsub.u32 uvrg, uvrg, uvrg_dy;                                                \
698   sub b, b, b_dy                                                               \
699
700 #define setup_spans_adjust_interpolants_down()                                 \
701   vadd.u32 uvrg, uvrg, uvrg_dy;                                                \
702   add b, b, b_dy                                                               \
703
704
705 #define setup_spans_clip_interpolants_increment()                              \
706   mla b, b_dy, clip, b;                                                        \
707   vmla.s32 uvrg, uvrg_dy, v_clip                                               \
708
709 #define setup_spans_clip_interpolants_decrement()                              \
710   mls b, b_dy, clip, b;                                                        \
711   vmls.s32 uvrg, uvrg_dy, v_clip                                               \
712
713 #define setup_spans_clip_alternate_yes()                                       \
714   smlal edge_alt_low, edge_alt_high, edge_dx_dy_alt, clip                      \
715
716 #define setup_spans_clip_alternate_no()                                        \
717
718 #define setup_spans_clip(direction, alternate_active)                          \
719   vdup.u32 v_clip, clip;                                                       \
720   setup_spans_clip_alternate_##alternate_active();                             \
721   setup_spans_clip_interpolants_##direction();                                 \
722   vmlal.s32 edges_xy, edges_dx_dy, v_clip_low                                  \
723
724
725 #define setup_spans_adjust_edges_alternate_no(left_index, right_index)         \
726   vmovl.s32 edge_shifts_64, edge_shifts;                                       \
727   vmovl.s32 edges_dx_dy_64, edges_dx_dy;                                       \
728                                                                                \
729   vshl.s64 edges_xy, edges_xy, edge_shifts_64;                                 \
730   vshl.s64 edges_dx_dy_64, edges_dx_dy_64, edge_shifts_64;                     \
731                                                                                \
732   vmov left_x_low, edges_xy_##left_index;                                      \
733   vmov right_x_low, edges_xy_##right_index;                                    \
734                                                                                \
735   vmov left_dx_dy_low, edges_dx_dy_64_##left_index;                            \
736   vmov left_dx_dy_high, edges_dx_dy_64_##left_index;                           \
737   vmov right_dx_dy_low, edges_dx_dy_64_##right_index;                          \
738   vmov right_dx_dy_high, edges_dx_dy_64_##right_index;                         \
739                                                                                \
740   vadd.u64 left_x_high, left_x_low, left_dx_dy_low;                            \
741   vadd.u64 right_x_high, right_x_low, right_dx_dy_low;                         \
742                                                                                \
743   vadd.u64 left_dx_dy, left_dx_dy, left_dx_dy;                                 \
744   vadd.u64 right_dx_dy, right_dx_dy, right_dx_dy                               \
745
746
747 #define setup_spans_adjust_edges_alternate_yes(left_index, right_index)        \
748   setup_spans_adjust_edges_alternate_no(left_index, right_index);              \
749                                                                                \
750   vdup.u16 y_mid_point, y_b;                                                   \
751   rsb temp, edge_shift_alt, #32;                                               \
752                                                                                \
753   lsl edge_alt_high, edge_alt_high, edge_shift_alt;                            \
754   orr edge_alt_high, edge_alt_high, edge_alt_low, lsr temp;                    \
755   lsl edge_alt_low, edge_alt_low, edge_shift_alt;                              \
756   vmov alternate_x_low, edge_alt_low, edge_alt_high;                           \
757                                                                                \
758   asr edge_dx_dy_alt_high, edge_dx_dy_alt, temp;                               \
759   lsl edge_dx_dy_alt_low, edge_dx_dy_alt, edge_shift_alt;                      \
760   vmov alternate_dx_dy_low, edge_dx_dy_alt_low, edge_dx_dy_alt_high;           \
761   vmov alternate_dx_dy_high, alternate_dx_dy_low;                              \
762                                                                                \
763   vadd.u64 alternate_x_high, alternate_x_low, alternate_dx_dy_low;             \
764   vadd.u64 alternate_dx_dy, alternate_dx_dy, alternate_dx_dy                   \
765
766
767 #define setup_spans_y_select_up()                                              \
768   vclt.s16 alternate_select, y_x4, y_mid_point                                 \
769
770 #define setup_spans_y_select_down()                                            \
771   vcgt.s16 alternate_select, y_x4, y_mid_point                                 \
772
773
774 #define setup_spans_alternate_select_left()                                    \
775   vbit.u16 left_right_x_16_low, alternate_x_16, alternate_select               \
776
777 #define setup_spans_alternate_select_right()                                   \
778   vbit.u16 left_right_x_16_high, alternate_x_16, alternate_select              \
779
780
781 #define setup_spans_set_x4_alternate_yes(alternate, direction)                 \
782   vshrn.s64 alternate_x_32_low, alternate_x, #32;                              \
783   vshrn.s64 left_x_32_low, left_x, #32;                                        \
784   vshrn.s64 right_x_32_low, right_x, #32;                                      \
785                                                                                \
786   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
787   vadd.u64 left_x, left_x, left_dx_dy;                                         \
788   vadd.u64 right_x, right_x, right_dx_dy;                                      \
789                                                                                \
790   vshrn.s64 alternate_x_32_high, alternate_x, #32;                             \
791   vshrn.s64 left_x_32_high, left_x, #32;                                       \
792   vshrn.s64 right_x_32_high, right_x, #32;                                     \
793                                                                                \
794   vadd.u64 alternate_x, alternate_x, alternate_dx_dy;                          \
795   vadd.u64 left_x, left_x, left_dx_dy;                                         \
796   vadd.u64 right_x, right_x, right_dx_dy;                                      \
797                                                                                \
798   vmovn.u32 alternate_x_16, alternate_x_32;                                    \
799   setup_spans_y_select_##direction();                                          \
800   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
801                                                                                \
802   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
803   setup_spans_alternate_select_##alternate();                                  \
804                                                                                \
805   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
806   str b, [ span_b_offset ], #4;                                                \
807   setup_spans_adjust_interpolants_##direction();                               \
808                                                                                \
809   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
810                                                                                \
811   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
812   str b, [ span_b_offset ], #4;                                                \
813   setup_spans_adjust_interpolants_##direction();                               \
814                                                                                \
815   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
816                                                                                \
817   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
818   str b, [ span_b_offset ], #4;                                                \
819   setup_spans_adjust_interpolants_##direction();                               \
820                                                                                \
821   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
822   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
823   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
824                                                                                \
825   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
826   str b, [ span_b_offset ], #4;                                                \
827   setup_spans_adjust_interpolants_##direction();                               \
828                                                                                \
829   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
830   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
831                                                                                \
832   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
833                                                                                \
834   setup_spans_adjust_y_##direction()                                           \
835
836
837 #define setup_spans_set_x4_alternate_no(alternate, direction)                  \
838   vshrn.s64 left_x_32_low, left_x, #32;                                        \
839   vshrn.s64 right_x_32_low, right_x, #32;                                      \
840                                                                                \
841   vadd.u64 left_x, left_x, left_dx_dy;                                         \
842   vadd.u64 right_x, right_x, right_dx_dy;                                      \
843                                                                                \
844   vshrn.s64 left_x_32_high, left_x, #32;                                       \
845   vshrn.s64 right_x_32_high, right_x, #32;                                     \
846                                                                                \
847   vadd.u64 left_x, left_x, left_dx_dy;                                         \
848   vadd.u64 right_x, right_x, right_dx_dy;                                      \
849                                                                                \
850   vmovn.u32 left_right_x_16_low, left_x_32;                                    \
851   vmovn.u32 left_right_x_16_high, right_x_32;                                  \
852                                                                                \
853   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
854   str b, [ span_b_offset ], #4;                                                \
855   setup_spans_adjust_interpolants_##direction();                               \
856                                                                                \
857   vmax.s16 left_right_x_16, left_right_x_16, left_edge;                        \
858                                                                                \
859   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
860   str b, [ span_b_offset ], #4;                                                \
861   setup_spans_adjust_interpolants_##direction();                               \
862                                                                                \
863   vmin.s16 left_right_x_16, left_right_x_16, right_edge;                       \
864                                                                                \
865   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
866   str b, [ span_b_offset ], #4;                                                \
867   setup_spans_adjust_interpolants_##direction();                               \
868                                                                                \
869   vsub.u16 left_right_x_16_high, left_right_x_16_high, left_right_x_16_low;    \
870   vadd.u16 left_right_x_16_high, left_right_x_16_high, c_0x0007;               \
871   vand.u16 span_shifts, left_right_x_16_high, c_0x0007;                        \
872                                                                                \
873   vst1.u32 { uvrg }, [ span_uvrg_offset, :128 ]!;                              \
874   str b, [ span_b_offset ], #4;                                                \
875   setup_spans_adjust_interpolants_##direction();                               \
876                                                                                \
877   vshl.u16 span_shifts, c_0xFFFE, span_shifts;                                 \
878   vshr.u16 left_right_x_16_high, left_right_x_16_high, #3;                     \
879                                                                                \
880   vst4.u16 { left_right_x_16, span_shifts_y }, [ span_edge_data ]!;            \
881                                                                                \
882   setup_spans_adjust_y_##direction()                                           \
883
884
885 #define edge_adjust_low           r11
886 #define edge_adjust_high          r12
887
888 #define setup_spans_alternate_adjust_yes()                                     \
889   smull edge_adjust_low, edge_adjust_high, edge_dx_dy_alt, height_minor_a;     \
890   subs edge_alt_low, edge_alt_low, edge_adjust_low;                            \
891   sbc edge_alt_high, edge_alt_high, edge_adjust_high                           \
892
893 #define setup_spans_alternate_adjust_no()                                      \
894
895
896 #define setup_spans_down(left_index, right_index, alternate, alternate_active) \
897   setup_spans_alternate_adjust_##alternate_active();                           \
898   setup_spans_load_b();                                                        \
899                                                                                \
900   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                     \
901   subs y_c, y_c, temp;                                                         \
902   subgt height, height, y_c;                                                   \
903   addgt height, height, #1;                                                    \
904                                                                                \
905   ldrsh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                   \
906   subs clip, temp, y_a;                                                        \
907   ble 0f;                                                                      \
908                                                                                \
909   sub height, height, clip;                                                    \
910   add y_a, y_a, clip;                                                          \
911   setup_spans_clip(increment, alternate_active);                               \
912                                                                                \
913  0:                                                                            \
914   cmp height, #0;                                                              \
915   ble 1f;                                                                      \
916                                                                                \
917   orr temp, y_a, y_a, lsl #16;                                                 \
918   add temp, temp, #(1 << 16);                                                  \
919   add y_a, temp, #2;                                                           \
920   add y_a, y_a, #(2 << 16);                                                    \
921   vmov.u32 y_x4, temp, y_a;                                                    \
922                                                                                \
923   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
924    right_index);                                                               \
925   setup_spans_prologue_b();                                                    \
926                                                                                \
927   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
928                                                                                \
929  2:                                                                            \
930   setup_spans_set_x4_alternate_##alternate_active(alternate, down);            \
931   subs height, height, #4;                                                     \
932   bhi 2b;                                                                      \
933                                                                                \
934  1:                                                                            \
935
936
937 #define setup_spans_alternate_pre_increment_yes()                              \
938   adds edge_alt_low, edge_alt_low, edge_dx_dy_alt;                             \
939   adc edge_alt_high, edge_alt_high, edge_dx_dy_alt, asr #31                    \
940
941 #define setup_spans_alternate_pre_increment_no()                               \
942
943
944 #define setup_spans_up_decrement_yes()                                         \
945   suble height, height, #1                                                     \
946
947 #define setup_spans_up_decrement_no()                                          \
948
949
950 #define setup_spans_up(left_index, right_index, alternate, alternate_active)   \
951   setup_spans_alternate_adjust_##alternate_active();                           \
952   setup_spans_load_b();                                                        \
953   sub y_a, y_a, #1;                                                            \
954                                                                                \
955   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ];                    \
956   subs temp, temp, y_c;                                                        \
957   subgt height, height, temp;                                                  \
958   setup_spans_up_decrement_##alternate_active();                               \
959                                                                                \
960   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ];                      \
961   subs clip, y_a, temp;                                                        \
962   ble 0f;                                                                      \
963                                                                                \
964   sub height, height, clip;                                                    \
965   sub y_a, y_a, clip;                                                          \
966   setup_spans_clip(decrement, alternate_active);                               \
967                                                                                \
968  0:                                                                            \
969   cmp height, #0;                                                              \
970   ble 1f;                                                                      \
971                                                                                \
972   orr temp, y_a, y_a, lsl #16;                                                 \
973   sub temp, temp, #(1 << 16);                                                  \
974   sub y_a, temp, #2;                                                           \
975   sub y_a, y_a, #(2 << 16);                                                    \
976   vmov.u32 y_x4, temp, y_a;                                                    \
977                                                                                \
978   vaddw.s32 edges_xy, edges_xy, edges_dx_dy;                                   \
979                                                                                \
980   setup_spans_alternate_pre_increment_##alternate_active();                    \
981   setup_spans_adjust_edges_alternate_##alternate_active(left_index,            \
982    right_index);                                                               \
983   setup_spans_adjust_interpolants_up();                                        \
984   setup_spans_prologue_b();                                                    \
985                                                                                \
986   strh height, [ psx_gpu, #psx_gpu_num_spans_offset ];                         \
987                                                                                \
988  2:                                                                            \
989   setup_spans_set_x4_alternate_##alternate_active(alternate, up);              \
990   subs height, height, #4;                                                     \
991   bhi 2b;                                                                      \
992                                                                                \
993  1:                                                                            \
994
995
996 #define setup_spans_epilogue()                                                 \
997   ldmia sp!, { r4 - r11, pc }                                                  \
998
999
1000 #define setup_spans_up_up(minor, major)                                        \
1001   setup_spans_prologue();                                                      \
1002   sub height_minor_a, y_a, y_b;                                                \
1003   sub height_minor_b, y_b, y_c;                                                \
1004   sub height, y_a, y_c;                                                        \
1005                                                                                \
1006   vdup.u32 x_starts, x_a;                                                      \
1007   vmov.u32 x_ends, x_c, x_b;                                                   \
1008                                                                                \
1009   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
1010   setup_spans_up(major, minor, minor, yes);                                    \
1011   setup_spans_epilogue()                                                       \
1012
1013 function(setup_spans_up_left)
1014   setup_spans_up_up(left, right)
1015
1016 function(setup_spans_up_right)
1017   setup_spans_up_up(right, left)
1018
1019
1020 #define setup_spans_down_down(minor, major)                                    \
1021   setup_spans_prologue();                                                      \
1022   sub height_minor_a, y_b, y_a;                                                \
1023   sub height_minor_b, y_c, y_b;                                                \
1024   sub height, y_c, y_a;                                                        \
1025                                                                                \
1026   vdup.u32 x_starts, x_a;                                                      \
1027   vmov.u32 x_ends, x_c, x_b;                                                   \
1028                                                                                \
1029   compute_edge_delta_x3(x_b, height_major, height_minor_a);                    \
1030   setup_spans_down(major, minor, minor, yes);                                  \
1031   setup_spans_epilogue()                                                       \
1032
1033 function(setup_spans_down_left)
1034   setup_spans_down_down(left, right)
1035
1036 function(setup_spans_down_right)
1037   setup_spans_down_down(right, left)
1038
1039
1040 #define setup_spans_up_flat()                                                  \
1041   sub height, y_a, y_c;                                                        \
1042                                                                                \
1043   compute_edge_delta_x2();                                                     \
1044   setup_spans_up(left, right, none, no);                                       \
1045   setup_spans_epilogue()                                                       \
1046
1047 function(setup_spans_up_a)
1048   setup_spans_prologue()
1049
1050   vmov.u32 x_starts, x_a, x_b
1051   vdup.u32 x_ends, x_c
1052
1053   setup_spans_up_flat()
1054
1055 function(setup_spans_up_b)
1056   setup_spans_prologue()
1057
1058   vdup.u32 x_starts, x_a
1059   vmov.u32 x_ends, x_b, x_c
1060
1061   setup_spans_up_flat()
1062
1063 #define setup_spans_down_flat()                                                \
1064   sub height, y_c, y_a;                                                        \
1065                                                                                \
1066   compute_edge_delta_x2();                                                     \
1067   setup_spans_down(left, right, none, no);                                     \
1068   setup_spans_epilogue()                                                       \
1069
1070 function(setup_spans_down_a)
1071   setup_spans_prologue()
1072
1073   vmov.u32 x_starts, x_a, x_b
1074   vdup.u32 x_ends, x_c
1075
1076   setup_spans_down_flat()
1077
1078 function(setup_spans_down_b)
1079   setup_spans_prologue()
1080
1081   vdup.u32 x_starts, x_a
1082   vmov.u32 x_ends, x_b, x_c
1083
1084   setup_spans_down_flat()
1085
1086
1087 #define middle_y                                          r9
1088
1089 #define edges_xy_b                                        q11
1090 #define edges_dx_dy_b                                     d26
1091 #define edge_shifts_b                                     d27
1092 #define edges_dx_dy_and_shifts_b                          q13
1093 #define height_increment                                  d20
1094
1095 #define edges_dx_dy_and_shifts                            q1
1096
1097 #define edges_xy_b_left                                   d22
1098 #define edges_xy_b_right                                  d23
1099
1100 #define setup_spans_up_down_load_edge_set_b()                                  \
1101   vmov edges_xy, edges_xy_b;                                                   \
1102   vmov edges_dx_dy_and_shifts, edges_dx_dy_and_shifts_b                        \
1103
1104
1105 function(setup_spans_up_down)
1106   setup_spans_prologue()
1107
1108   // s32 middle_y = y_a;
1109   sub height_minor_a, y_a, y_b
1110   sub height_minor_b, y_c, y_a
1111   sub height_major, y_c, y_b
1112
1113   vmov.u32 x_starts, x_a, x_c
1114   vdup.u32 x_ends, x_b
1115
1116   compute_edge_delta_x3(x_a, height_minor_a, height_major)
1117
1118   mov temp, #0
1119   vmov.u32 height_increment, temp, height_minor_b
1120   vmlal.s32 edges_xy, edges_dx_dy, height_increment
1121
1122   vmov edges_xy_b_left, edge_alt_low, edge_alt_high
1123   vmov edges_xy_b_right, edges_xy_right
1124
1125   vmov edge_shifts_b, edge_shifts
1126   vmov.u32 edge_shifts_b[0], edge_shift_alt
1127
1128   vneg.s32 edges_dx_dy_b, edges_dx_dy
1129   vmov.u32 edges_dx_dy_b[0], edge_dx_dy_alt
1130
1131   mov middle_y, y_a
1132   
1133   setup_spans_load_b()
1134   sub y_a, y_a, #1
1135
1136   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1137   subs temp, temp, y_b
1138   subgt height_minor_a, height_minor_a, temp
1139
1140   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1141   subs clip, y_a, temp
1142   ble 0f
1143
1144   sub height_minor_a, height_minor_a, clip
1145   sub y_a, y_a, clip
1146   setup_spans_clip(decrement, no)
1147
1148  0:                                                                
1149   cmp height_minor_a, #0
1150   ble 3f
1151
1152   orr temp, y_a, y_a, lsl #16
1153   sub temp, temp, #(1 << 16)
1154   sub y_a, temp, #2
1155   sub y_a, y_a, #(2 << 16)
1156   vmov.u32 y_x4, temp, y_a
1157
1158   vaddw.s32 edges_xy, edges_xy, edges_dx_dy
1159
1160   strh height_minor_a, [ psx_gpu, #psx_gpu_num_spans_offset ]
1161
1162   setup_spans_adjust_edges_alternate_no(left, right); 
1163   setup_spans_adjust_interpolants_up()
1164   setup_spans_up_down_load_edge_set_b()
1165
1166   setup_spans_prologue_b()
1167
1168
1169  2: 
1170   setup_spans_set_x4_alternate_no(none, up)
1171   subs height_minor_a, height_minor_a, #4
1172   bhi 2b
1173
1174   add span_edge_data, span_edge_data, height_minor_a, lsl #3
1175   add span_uvrg_offset, span_uvrg_offset, height_minor_a, lsl #4
1176   add span_b_offset, span_b_offset, height_minor_a, lsl #2
1177
1178  4:
1179   add temp, psx_gpu, #psx_gpu_uvrg_offset
1180   vld1.32 { uvrg }, [ temp ]
1181   mov y_a, middle_y
1182   
1183   setup_spans_load_b()
1184
1185   ldrh temp, [ psx_gpu, #psx_gpu_viewport_end_y_offset ]
1186   subs y_c, y_c, temp
1187   subgt height_minor_b, height_minor_b, y_c
1188   addgt height_minor_b, height_minor_b, #1
1189
1190   ldrh temp, [ psx_gpu, #psx_gpu_viewport_start_y_offset ]
1191   subs clip, temp, y_a
1192   ble 0f
1193
1194   sub height_minor_b, height_minor_b, clip
1195   add y_a, y_a, clip
1196   setup_spans_clip(increment, no)
1197
1198  0:
1199   cmp height_minor_b, #0
1200   ble 1f
1201
1202   orr temp, y_a, y_a, lsl #16
1203   add temp, temp, #(1 << 16) 
1204   add y_a, temp, #2
1205   add y_a, y_a, #(2 << 16)
1206   vmov.u32 y_x4, temp, y_a
1207
1208   setup_spans_adjust_edges_alternate_no(left, right)
1209
1210   ldrh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1211   add temp, temp, height_minor_b
1212   strh temp, [ psx_gpu, #psx_gpu_num_spans_offset ]
1213
1214  2:                                                     
1215   setup_spans_set_x4_alternate_no(none, down)
1216   subs height_minor_b, height_minor_b, #4
1217   bhi 2b
1218
1219  1:
1220   setup_spans_epilogue()
1221
1222  3:
1223   setup_spans_up_down_load_edge_set_b()
1224   setup_spans_prologue_b()
1225   bal 4b
1226
1227
1228 #undef span_uvrg_offset
1229 #undef span_edge_data
1230 #undef span_b_offset
1231 #undef left_x
1232 #undef b
1233
1234 #define psx_gpu                                  r0
1235 #define num_spans                                r1
1236 #define span_uvrg_offset                         r2
1237 #define span_edge_data                           r3
1238 #define span_b_offset                            r4
1239 #define b_dx                                     r5
1240 #define span_num_blocks                          r6
1241 #define y                                        r7
1242 #define left_x                                   r8
1243 #define b                                        r9
1244 #define dither_offset_ptr                        r10
1245 #define block_ptr_a                              r11
1246 #define fb_ptr                                   r12
1247 #define num_blocks                               r14
1248
1249 #define uvrg_dx_ptr                              r2
1250 #define texture_mask_ptr                         r3
1251 #define dither_shift                             r8
1252 #define dither_row                               r10
1253
1254 #define c_32                                     r7
1255 #define b_dx4                                    r8
1256 #define b_dx8                                    r9
1257 #define block_ptr_b                              r10
1258
1259 #define block_span_ptr                           r10
1260 #define right_mask                               r8
1261
1262 #define color                                    r2
1263 #define color_r                                  r3
1264 #define color_g                                  r4
1265 #define color_b                                  r5
1266
1267 #undef uvrg
1268
1269 #define u_block                                  q0
1270 #define v_block                                  q1
1271 #define r_block                                  q2
1272 #define g_block                                  q3
1273 #define b_block                                  q4
1274
1275 #define uv_dx4                                   d10
1276 #define rg_dx4                                   d11
1277 #define uv_dx8                                   d12
1278 #define rg_dx8                                   d13
1279 #define b_whole_8                                d14
1280 #define fb_mask_ptrs                             d15
1281
1282 #define uvrg_dx4                                 q5
1283 #define uvrg_dx8                                 q6
1284 #define uv_dx8                                   d12
1285 #define rg_dx8                                   d13
1286
1287 #define u_whole                                  q8
1288 #define v_whole                                  q9
1289 #define r_whole                                  q10
1290 #define g_whole                                  q11
1291 #define b_whole                                  q12
1292
1293 #define u_whole_low                              d16
1294 #define u_whole_high                             d17
1295 #define v_whole_low                              d18
1296 #define v_whole_high                             d19
1297 #define r_whole_low                              d20
1298 #define r_whole_high                             d21
1299 #define g_whole_low                              d22
1300 #define g_whole_high                             d23
1301 #define b_whole_low                              d24
1302 #define b_whole_high                             d25
1303
1304 #define dx4                                      q13
1305 #define dx8                                      q13
1306
1307 #define u_whole_8                                d26
1308 #define v_whole_8                                d27
1309 #define u_whole_8b                               d24
1310 #define r_whole_8                                d24
1311 #define g_whole_8                                d25
1312
1313 #define uv_whole_8                               q13
1314 #define uv_whole_8b                              q14
1315
1316 #define dither_offsets                           q14
1317 #define texture_mask                             q15
1318 #define texture_mask_u                           d30
1319 #define texture_mask_v                           d31
1320
1321 #define dither_offsets_short                     d28
1322
1323 #define v_left_x                                 q8
1324 #define uvrg                                     q9
1325 #define block_span                               q10
1326
1327 #define uv                                       d18
1328 #define rg                                       d19
1329
1330 #define draw_mask                                q1
1331 #define draw_mask_edge                           q13
1332 #define test_mask                                q0
1333
1334 #define uvrg_dx                                  q3
1335
1336 #define colors                                   q2
1337
1338 #define setup_blocks_texture_swizzled()                                        \
1339   vand.u8 u_whole_8b, u_whole_8, texture_mask_u;                               \
1340   vsli.u8 u_whole_8, v_whole_8, #4;                                            \
1341   vsri.u8 v_whole_8, u_whole_8b, #4                                            \
1342
1343 #define setup_blocks_texture_unswizzled()                                      \
1344
1345
1346 #define setup_blocks_shaded_textured_builder(swizzling)                        \
1347 .align 3;                                                                      \
1348                                                                                \
1349 function(setup_blocks_shaded_textured_dithered_##swizzling##_indirect)         \
1350   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1351   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1352                                                                                \
1353   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1354   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1355                                                                                \
1356   cmp num_spans, #0;                                                           \
1357   bxeq lr;                                                                     \
1358                                                                                \
1359   stmdb sp!, { r4 - r11, r14 };                                                \
1360   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1361                                                                                \
1362   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
1363   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1364                                                                                \
1365   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1366   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1367                                                                                \
1368   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1369   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1370                                                                                \
1371   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
1372   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1373                                                                                \
1374   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1375                                                                                \
1376  0:                                                                            \
1377   vmov.u8 fb_mask_ptrs, #0;                                                    \
1378                                                                                \
1379   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1380   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1381                                                                                \
1382   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1383   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1384                                                                                \
1385   cmp span_num_blocks, #0;                                                     \
1386   beq 1f;                                                                      \
1387                                                                                \
1388   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1389   add num_blocks, span_num_blocks, num_blocks;                                 \
1390                                                                                \
1391   cmp num_blocks, #MAX_BLOCKS;                                                 \
1392   bgt 2f;                                                                      \
1393                                                                                \
1394  3:                                                                            \
1395   ldr b, [ span_b_offset ];                                                    \
1396   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1397                                                                                \
1398   vdup.u32 v_left_x, left_x;                                                   \
1399   and y, y, #0x3;                                                              \
1400                                                                                \
1401   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1402   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1403                                                                                \
1404   mla b, b_dx, left_x, b;                                                      \
1405   and dither_shift, left_x, #0x03;                                             \
1406                                                                                \
1407   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1408   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1409                                                                                \
1410   mov dither_shift, dither_shift, lsl #3;                                      \
1411   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1412                                                                                \
1413   mov c_32, #32;                                                               \
1414   subs span_num_blocks, span_num_blocks, #1;                                   \
1415                                                                                \
1416   mov dither_row, dither_row, ror dither_shift;                                \
1417   mov b_dx4, b_dx, lsl #2;                                                     \
1418                                                                                \
1419   vdup.u32 dither_offsets_short, dither_row;                                   \
1420   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1421                                                                                \
1422   vdup.u32 b_block, b;                                                         \
1423   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1424                                                                                \
1425   vdup.u32 u_block, uv[0];                                                     \
1426   mov b_dx8, b_dx, lsl #3;                                                     \
1427                                                                                \
1428   vdup.u32 v_block, uv[1];                                                     \
1429   vdup.u32 r_block, rg[0];                                                     \
1430   vdup.u32 g_block, rg[1];                                                     \
1431                                                                                \
1432   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1433                                                                                \
1434   vadd.u32 u_block, u_block, block_span;                                       \
1435   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1436                                                                                \
1437   vadd.u32 v_block, v_block, block_span;                                       \
1438   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1439                                                                                \
1440   vadd.u32 r_block, r_block, block_span;                                       \
1441   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1442                                                                                \
1443   vadd.u32 g_block, g_block, block_span;                                       \
1444   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
1445                                                                                \
1446   vadd.u32 b_block, b_block, block_span;                                       \
1447   add block_ptr_b, block_ptr_a, #16;                                           \
1448                                                                                \
1449   vshrn.u32 u_whole_low, u_block, #16;                                         \
1450   vshrn.u32 v_whole_low, v_block, #16;                                         \
1451   vshrn.u32 r_whole_low, r_block, #16;                                         \
1452   vshrn.u32 g_whole_low, g_block, #16;                                         \
1453                                                                                \
1454   vdup.u32 dx4, uv_dx4[0];                                                     \
1455   vshrn.u32 b_whole_low, b_block, #16;                                         \
1456                                                                                \
1457   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1458   vdup.u32 dx4, uv_dx4[1];                                                     \
1459                                                                                \
1460   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1461   vdup.u32 dx4, rg_dx4[0];                                                     \
1462                                                                                \
1463   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1464   vdup.u32 dx4, rg_dx4[1];                                                     \
1465                                                                                \
1466   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1467   vdup.u32 dx4, b_dx4;                                                         \
1468                                                                                \
1469   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1470   vdup.u32 dx8, uv_dx8[0];                                                     \
1471                                                                                \
1472   vadd.u32 u_block, u_block, dx8;                                              \
1473   vdup.u32 dx8, uv_dx8[1];                                                     \
1474                                                                                \
1475   vadd.u32 v_block, v_block, dx8;                                              \
1476   vdup.u32 dx8, rg_dx8[0];                                                     \
1477                                                                                \
1478   vadd.u32 r_block, r_block, dx8;                                              \
1479   vdup.u32 dx8, rg_dx8[1];                                                     \
1480                                                                                \
1481   vadd.u32 g_block, g_block, dx8;                                              \
1482   vdup.u32 dx8, b_dx8;                                                         \
1483                                                                                \
1484   vadd.u32 b_block, b_block, dx8;                                              \
1485   vmovn.u16 u_whole_8, u_whole;                                                \
1486                                                                                \
1487   vmovn.u16 v_whole_8, v_whole;                                                \
1488                                                                                \
1489   vmovn.u16 b_whole_8, b_whole;                                                \
1490   pld [ fb_ptr ];                                                              \
1491   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1492                                                                                \
1493   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1494   setup_blocks_texture_##swizzling();                                          \
1495                                                                                \
1496   vmovn.u16 r_whole_8, r_whole;                                                \
1497   beq 5f;                                                                      \
1498                                                                                \
1499  4:                                                                            \
1500   vmovn.u16 g_whole_8, g_whole;                                                \
1501   vshrn.u32 u_whole_low, u_block, #16;                                         \
1502                                                                                \
1503   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1504   vshrn.u32 v_whole_low, v_block, #16;                                         \
1505                                                                                \
1506   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1507   vshrn.u32 r_whole_low, r_block, #16;                                         \
1508                                                                                \
1509   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1510   vshrn.u32 g_whole_low, g_block, #16;                                         \
1511                                                                                \
1512   vdup.u32 dx4, uv_dx4[0];                                                     \
1513   vshrn.u32 b_whole_low, b_block, #16;                                         \
1514                                                                                \
1515   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1516   vdup.u32 dx4, uv_dx4[1];                                                     \
1517                                                                                \
1518   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1519   vdup.u32 dx4, rg_dx4[0];                                                     \
1520                                                                                \
1521   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
1522   vdup.u32 dx4, rg_dx4[1];                                                     \
1523                                                                                \
1524   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
1525   vdup.u32 dx4, b_dx4;                                                         \
1526                                                                                \
1527   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
1528   vdup.u32 dx8, uv_dx8[0];                                                     \
1529                                                                                \
1530   vadd.u32 u_block, u_block, dx8;                                              \
1531   vdup.u32 dx8, uv_dx8[1];                                                     \
1532                                                                                \
1533   vadd.u32 v_block, v_block, dx8;                                              \
1534   vdup.u32 dx8, rg_dx8[0];                                                     \
1535                                                                                \
1536   vadd.u32 r_block, r_block, dx8;                                              \
1537   vdup.u32 dx8, rg_dx8[1];                                                     \
1538                                                                                \
1539   vadd.u32 g_block, g_block, dx8;                                              \
1540   vdup.u32 dx8, b_dx8;                                                         \
1541                                                                                \
1542   vadd.u32 b_block, b_block, dx8;                                              \
1543   vmovn.u16 u_whole_8, u_whole;                                                \
1544                                                                                \
1545   add fb_ptr, fb_ptr, #16;                                                     \
1546   vmovn.u16 v_whole_8, v_whole;                                                \
1547                                                                                \
1548   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1549   vmovn.u16 b_whole_8, b_whole;                                                \
1550                                                                                \
1551   pld [ fb_ptr ];                                                              \
1552                                                                                \
1553   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1554   subs span_num_blocks, span_num_blocks, #1;                                   \
1555                                                                                \
1556   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1557   setup_blocks_texture_##swizzling();                                          \
1558                                                                                \
1559   vmovn.u16 r_whole_8, r_whole;                                                \
1560   bne 4b;                                                                      \
1561                                                                                \
1562  5:                                                                            \
1563   vmovn.u16 g_whole_8, g_whole;                                                \
1564   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1565                                                                                \
1566   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1567   vdup.u8 draw_mask, right_mask;                                               \
1568                                                                                \
1569   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1570   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1571   vzip.u8 u_whole_8, v_whole_8;                                                \
1572                                                                                \
1573   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1574   vst1.u32 { r_whole_8, g_whole_8 }, [ block_ptr_b, :128 ], c_32;              \
1575   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1576   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1577   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1578                                                                                \
1579  1:                                                                            \
1580   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1581   add span_b_offset, span_b_offset, #4;                                        \
1582                                                                                \
1583   add span_edge_data, span_edge_data, #8;                                      \
1584   subs num_spans, num_spans, #1;                                               \
1585                                                                                \
1586   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1587   bne 0b;                                                                      \
1588                                                                                \
1589   ldmia sp!, { r4 - r11, pc };                                                 \
1590                                                                                \
1591  2:                                                                            \
1592   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1593   vpush { texture_mask };                                                      \
1594   vpush { uvrg_dx4 };                                                          \
1595                                                                                \
1596   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1597   bl flush_render_block_buffer;                                                \
1598   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1599                                                                                \
1600   vpop { uvrg_dx4 };                                                           \
1601   vpop { texture_mask };                                                       \
1602                                                                                \
1603   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1604   vmov.u8 fb_mask_ptrs, #0;                                                    \
1605                                                                                \
1606   mov num_blocks, span_num_blocks;                                             \
1607   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1608   bal 3b                                                                       \
1609
1610
1611 setup_blocks_shaded_textured_builder(swizzled)
1612 setup_blocks_shaded_textured_builder(unswizzled)
1613
1614
1615 #define setup_blocks_unshaded_textured_builder(swizzling)                      \
1616 .align 3;                                                                      \
1617                                                                                \
1618 function(setup_blocks_unshaded_textured_dithered_##swizzling##_indirect)       \
1619   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
1620   add uvrg_dx_ptr, psx_gpu, #psx_gpu_uvrg_dx_offset;                           \
1621                                                                                \
1622   vld1.u32 { uvrg_dx }, [ uvrg_dx_ptr, :128 ];                                 \
1623   add texture_mask_ptr, psx_gpu, #psx_gpu_texture_mask_width_offset;           \
1624                                                                                \
1625   cmp num_spans, #0;                                                           \
1626   bxeq lr;                                                                     \
1627                                                                                \
1628   stmdb sp!, { r4 - r11, r14 };                                                \
1629   vshl.u32 uvrg_dx4, uvrg_dx, #2;                                              \
1630                                                                                \
1631   vshl.u32 uvrg_dx8, uvrg_dx, #3;                                              \
1632                                                                                \
1633   vld2.u8 { texture_mask_u[], texture_mask_v[] }, [ texture_mask_ptr, :16 ];   \
1634   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
1635                                                                                \
1636   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1637   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
1638                                                                                \
1639   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1640                                                                                \
1641   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
1642                                                                                \
1643  0:                                                                            \
1644   vmov.u8 fb_mask_ptrs, #0;                                                    \
1645                                                                                \
1646   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
1647   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
1648                                                                                \
1649   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
1650   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
1651                                                                                \
1652   cmp span_num_blocks, #0;                                                     \
1653   beq 1f;                                                                      \
1654                                                                                \
1655   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
1656   add num_blocks, span_num_blocks, num_blocks;                                 \
1657                                                                                \
1658   cmp num_blocks, #MAX_BLOCKS;                                                 \
1659   bgt 2f;                                                                      \
1660                                                                                \
1661  3:                                                                            \
1662   add fb_ptr, fb_ptr, y, lsl #11;                                              \
1663                                                                                \
1664   vdup.u32 v_left_x, left_x;                                                   \
1665   and y, y, #0x3;                                                              \
1666                                                                                \
1667   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
1668   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
1669                                                                                \
1670   and dither_shift, left_x, #0x03;                                             \
1671                                                                                \
1672   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
1673   vshr.u32 uvrg_dx, uvrg_dx4, #2;                                              \
1674                                                                                \
1675   mov dither_shift, dither_shift, lsl #3;                                      \
1676   vmla.u32 uvrg, uvrg_dx, v_left_x;                                            \
1677                                                                                \
1678   mov c_32, #32;                                                               \
1679   subs span_num_blocks, span_num_blocks, #1;                                   \
1680                                                                                \
1681   mov dither_row, dither_row, ror dither_shift;                                \
1682                                                                                \
1683   vdup.u32 dither_offsets_short, dither_row;                                   \
1684   add block_span_ptr, psx_gpu, #psx_gpu_u_block_span_offset;                   \
1685                                                                                \
1686   vshll.s8 dither_offsets, dither_offsets_short, #4;                           \
1687                                                                                \
1688   vdup.u32 u_block, uv[0];                                                     \
1689                                                                                \
1690   vdup.u32 v_block, uv[1];                                                     \
1691   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1692                                                                                \
1693   vadd.u32 u_block, u_block, block_span;                                       \
1694   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
1695                                                                                \
1696   vadd.u32 v_block, v_block, block_span;                                       \
1697   add block_ptr_b, block_ptr_a, #16;                                           \
1698                                                                                \
1699   vshrn.u32 u_whole_low, u_block, #16;                                         \
1700   vshrn.u32 v_whole_low, v_block, #16;                                         \
1701                                                                                \
1702   vdup.u32 dx4, uv_dx4[0];                                                     \
1703                                                                                \
1704   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1705   vdup.u32 dx4, uv_dx4[1];                                                     \
1706                                                                                \
1707   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1708   vdup.u32 dx8, uv_dx8[0];                                                     \
1709                                                                                \
1710   vadd.u32 u_block, u_block, dx8;                                              \
1711   vdup.u32 dx8, uv_dx8[1];                                                     \
1712                                                                                \
1713   vadd.u32 v_block, v_block, dx8;                                              \
1714   vmovn.u16 u_whole_8, u_whole;                                                \
1715                                                                                \
1716   vmovn.u16 v_whole_8, v_whole;                                                \
1717                                                                                \
1718   pld [ fb_ptr ];                                                              \
1719   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1720                                                                                \
1721   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1722   setup_blocks_texture_##swizzling();                                          \
1723                                                                                \
1724   beq 5f;                                                                      \
1725                                                                                \
1726  4:                                                                            \
1727   vshrn.u32 u_whole_low, u_block, #16;                                         \
1728                                                                                \
1729   vst2.u8 { u_whole_8, v_whole_8 }, [ block_ptr_a, :128 ], c_32;               \
1730   vshrn.u32 v_whole_low, v_block, #16;                                         \
1731                                                                                \
1732   add block_ptr_b, block_ptr_b, #32;                                           \
1733   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1734                                                                                \
1735   vdup.u32 dx4, uv_dx4[0];                                                     \
1736   vaddhn.u32 u_whole_high, u_block, dx4;                                       \
1737   vdup.u32 dx4, uv_dx4[1];                                                     \
1738                                                                                \
1739   vaddhn.u32 v_whole_high, v_block, dx4;                                       \
1740   vdup.u32 dx8, uv_dx8[0];                                                     \
1741                                                                                \
1742   vadd.u32 u_block, u_block, dx8;                                              \
1743   vdup.u32 dx8, uv_dx8[1];                                                     \
1744                                                                                \
1745   vadd.u32 v_block, v_block, dx8;                                              \
1746   vmovn.u16 u_whole_8, u_whole;                                                \
1747                                                                                \
1748   add fb_ptr, fb_ptr, #16;                                                     \
1749   vmovn.u16 v_whole_8, v_whole;                                                \
1750                                                                                \
1751   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1752   pld [ fb_ptr ];                                                              \
1753                                                                                \
1754   vmov.u32 fb_mask_ptrs[1], fb_ptr;                                            \
1755   subs span_num_blocks, span_num_blocks, #1;                                   \
1756                                                                                \
1757   vand.u8 uv_whole_8, uv_whole_8, texture_mask;                                \
1758   setup_blocks_texture_##swizzling();                                          \
1759                                                                                \
1760   bne 4b;                                                                      \
1761                                                                                \
1762  5:                                                                            \
1763   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ];           \
1764                                                                                \
1765   vld1.u32 { test_mask }, [ psx_gpu, :128 ];                                   \
1766   vdup.u8 draw_mask, right_mask;                                               \
1767                                                                                \
1768   vmov.u32 fb_mask_ptrs[0], right_mask;                                        \
1769   vtst.u16 draw_mask, draw_mask, test_mask;                                    \
1770   vzip.u8 u_whole_8, v_whole_8;                                                \
1771                                                                                \
1772   vbic.u16 uv_whole_8, uv_whole_8, draw_mask;                                  \
1773   add block_ptr_b, block_ptr_b, #32;                                           \
1774   vst1.u32 { uv_whole_8 }, [ block_ptr_a, :128 ], c_32;                        \
1775   vst1.u32 { dither_offsets }, [ block_ptr_b, :128 ], c_32;                    \
1776   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32;           \
1777                                                                                \
1778  1:                                                                            \
1779   add span_uvrg_offset, span_uvrg_offset, #16;                                 \
1780   add span_edge_data, span_edge_data, #8;                                      \
1781   subs num_spans, num_spans, #1;                                               \
1782                                                                                \
1783   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
1784   bne 0b;                                                                      \
1785                                                                                \
1786   ldmia sp!, { r4 - r11, pc };                                                 \
1787                                                                                \
1788  2:                                                                            \
1789   /* TODO: Load from psx_gpu instead of saving/restoring these               */\
1790   vpush { texture_mask };                                                      \
1791   vpush { uvrg_dx4 };                                                          \
1792                                                                                \
1793   stmdb sp!, { r0 - r3, r12, r14 };                                            \
1794   bl flush_render_block_buffer;                                                \
1795   ldmia sp!, { r0 - r3, r12, r14 };                                            \
1796                                                                                \
1797   vpop { uvrg_dx4 };                                                           \
1798   vpop { texture_mask };                                                       \
1799                                                                                \
1800   vadd.u32 uvrg_dx8, uvrg_dx4, uvrg_dx4;                                       \
1801   vmov.u8 fb_mask_ptrs, #0;                                                    \
1802                                                                                \
1803   mov num_blocks, span_num_blocks;                                             \
1804   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
1805   bal 3b                                                                       \
1806
1807
1808 setup_blocks_unshaded_textured_builder(swizzled)
1809 setup_blocks_unshaded_textured_builder(unswizzled)
1810
1811
1812 .align 3
1813
1814 function(setup_blocks_unshaded_untextured_undithered_unswizzled_indirect)
1815   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1816   veor.u32 draw_mask, draw_mask, draw_mask
1817
1818   cmp num_spans, #0
1819   bxeq lr
1820
1821   stmdb sp!, { r4 - r11, r14 }
1822   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1823
1824   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1825
1826   ubfx color_r, color, #3, #5
1827   ubfx color_g, color, #11, #5
1828   ubfx color_b, color, #19, #5
1829
1830   orr color, color_r, color_b, lsl #10
1831   orr color, color, color_g, lsl #5
1832
1833   vdup.u16 colors, color
1834
1835   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1836   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1837
1838   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1839   add block_ptr_a, block_ptr_a, num_blocks, lsl #6
1840
1841  0:
1842   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1843   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1844
1845   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1846
1847   cmp span_num_blocks, #0
1848   beq 1f
1849
1850   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1851   add num_blocks, span_num_blocks, num_blocks
1852
1853   cmp num_blocks, #MAX_BLOCKS
1854   bgt 2f
1855
1856  3:
1857   add fb_ptr, fb_ptr, y, lsl #11
1858   and y, y, #0x3
1859
1860   add fb_ptr, fb_ptr, left_x, lsl #1
1861   mov c_32, #32
1862
1863   subs span_num_blocks, span_num_blocks, #1
1864
1865   add block_ptr_b, block_ptr_a, #16
1866   pld [ fb_ptr ]
1867
1868   vmov.u32 fb_mask_ptrs[1], fb_ptr
1869   beq 5f
1870
1871  4:
1872   vst1.u32 { draw_mask }, [ block_ptr_a, :128 ], c_32
1873   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1874   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1875
1876   add fb_ptr, fb_ptr, #16
1877   add block_ptr_b, block_ptr_b, #32
1878
1879   pld [ fb_ptr ]
1880
1881   vmov.u32 fb_mask_ptrs[1], fb_ptr
1882   subs span_num_blocks, span_num_blocks, #1
1883
1884   bne 4b
1885
1886  5:
1887   ldrh right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1888
1889   vdup.u8 draw_mask_edge, right_mask
1890   vtst.u16 draw_mask_edge, draw_mask_edge, test_mask
1891
1892   vst1.u32 { colors }, [ block_ptr_b, :128 ], c_32
1893   vst1.u32 { draw_mask_edge }, [ block_ptr_a, :128 ], c_32
1894   add block_ptr_b, block_ptr_b, #32
1895   vst1.u32 { b_whole_8, fb_mask_ptrs }, [ block_ptr_a, :128 ], c_32
1896
1897  1:
1898   add span_edge_data, span_edge_data, #8
1899   subs num_spans, num_spans, #1
1900
1901   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1902   bne 0b
1903
1904   ldmia sp!, { r4 - r11, pc }
1905                                                                            
1906  2:
1907   vpush { colors }
1908
1909   stmdb sp!, { r0 - r3, r12, r14 }
1910   bl flush_render_block_buffer
1911   ldmia sp!, { r0 - r3, r12, r14 }
1912
1913   vpop { colors }
1914
1915   vld1.u32 { test_mask }, [ psx_gpu, :128 ]
1916   veor.u32 draw_mask, draw_mask, draw_mask
1917
1918   mov num_blocks, span_num_blocks
1919   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset
1920   bal 3b
1921
1922
1923 #define mask_msb_scalar                                   r14
1924
1925 #define msb_mask                                          q15
1926
1927 #define pixels_low                                        d16
1928
1929 #define msb_mask_low                                      d30
1930 #define msb_mask_high                                     d31
1931
1932
1933 .align 3
1934
1935 function(setup_blocks_unshaded_untextured_undithered_unswizzled_direct)
1936   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ]
1937
1938   cmp num_spans, #0
1939   bxeq lr
1940
1941   stmdb sp!, { r4 - r11, r14 }
1942
1943   ldr color, [ psx_gpu, #psx_gpu_triangle_color_offset ]
1944
1945   ubfx color_r, color, #3, #5
1946   ubfx color_g, color, #11, #5
1947
1948   ldrh mask_msb_scalar, [ psx_gpu, #psx_gpu_mask_msb_offset ]
1949   ubfx color_b, color, #19, #5
1950
1951   orr color, color_r, color_b, lsl #10
1952   orr color, color, color_g, lsl #5
1953   orr color, color, mask_msb_scalar
1954
1955   vdup.u16 colors, color
1956
1957   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset
1958
1959  0:
1960   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ]
1961   ldrh y, [ span_edge_data, #edge_data_y_offset ]
1962
1963   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ]
1964
1965   cmp span_num_blocks, #0
1966   beq 1f
1967
1968   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ]
1969
1970   add fb_ptr, fb_ptr, y, lsl #11
1971   subs span_num_blocks, span_num_blocks, #1
1972
1973   add fb_ptr, fb_ptr, left_x, lsl #1
1974   beq 3f
1975
1976  2:
1977   vst1.u32 { colors }, [ fb_ptr ]!
1978   subs span_num_blocks, span_num_blocks, #1
1979
1980   bne 2b
1981
1982  3:
1983   ldrb right_mask, [ span_edge_data, #edge_data_right_mask_offset ]
1984   eor right_mask, right_mask, #0xFF
1985
1986  4:
1987   strh color, [ fb_ptr ], #2
1988   movs right_mask, right_mask, lsr #1
1989   bne 4b
1990
1991  1:
1992   add span_edge_data, span_edge_data, #8
1993   subs num_spans, num_spans, #1
1994
1995   strh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ]
1996   bne 0b
1997
1998   ldmia sp!, { r4 - r11, pc }
1999                                                                            
2000
2001
2002 #undef c_64
2003
2004 #define c_64                                              r7
2005 #define rg_dx_ptr                                         r2
2006
2007
2008 #undef r_block
2009 #undef g_block
2010 #undef b_block
2011 #undef r_whole
2012 #undef g_whole
2013 #undef b_whole
2014 #undef r_whole_low
2015 #undef r_whole_high
2016 #undef g_whole_low
2017 #undef g_whole_high
2018 #undef b_whole_low
2019 #undef b_whole_high
2020 #undef r_whole_8
2021 #undef g_whole_8
2022 #undef b_whole_8
2023 #undef dither_offsets
2024 #undef rg_dx4
2025 #undef rg_dx8
2026 #undef dx4
2027 #undef dx8
2028 #undef v_left_x
2029 #undef uvrg
2030 #undef block_span
2031 #undef rg
2032 #undef draw_mask
2033 #undef test_mask
2034
2035 #define r_block                                           q0
2036 #define g_block                                           q1
2037 #define b_block                                           q2
2038
2039 #define r_whole                                           q3
2040 #define g_whole                                           q4
2041 #define b_whole                                           q5
2042
2043 #define r_whole_low                                       d6
2044 #define r_whole_high                                      d7
2045 #define g_whole_low                                       d8
2046 #define g_whole_high                                      d9
2047 #define b_whole_low                                       d10
2048 #define b_whole_high                                      d11
2049
2050 #define gb_whole_8                                        q6
2051
2052 #define g_whole_8                                         d12
2053 #define b_whole_8                                         d13
2054
2055 #define r_whole_8                                         d14
2056
2057 #define pixels                                            q8
2058
2059 #define rg_dx4                                            d18
2060 #define rg_dx8                                            d19
2061
2062 #define dx4                                               q10
2063 #define dx8                                               q10
2064
2065 #define v_left_x                                          d6
2066 #define uvrg                                              q4
2067 #define block_span                                        q5
2068
2069 #define rg                                                d9
2070
2071 #define d64_1                                             d22
2072 #define d64_128                                           d23
2073
2074 #define d128_4                                            q12
2075 #define d128_0x7                                          q13
2076
2077 #define d64_4                                             d24
2078
2079 #define dither_offsets                                    q14
2080 #define draw_mask                                         q15
2081
2082 #define dither_offsets_low                                d28
2083
2084 #define rg_dx                                             d0
2085 #define test_mask                                         q10
2086
2087
2088 #define setup_blocks_shaded_untextured_dither_a_dithered()                     \
2089   vqadd.u8 r_whole_8, r_whole_8, dither_offsets_low;                           \
2090   vqadd.u8 gb_whole_8, gb_whole_8, dither_offsets;                             \
2091
2092 #define setup_blocks_shaded_untextured_dither_b_dithered()                     \
2093   vqsub.u8 r_whole_8, r_whole_8, d64_4;                                        \
2094   vqsub.u8 gb_whole_8, gb_whole_8, d128_4                                      \
2095
2096 #define setup_blocks_shaded_untextured_dither_a_undithered()                   \
2097
2098 #define setup_blocks_shaded_untextured_dither_b_undithered()                   \
2099
2100
2101 #define setup_blocks_shaded_untextured_indirect_builder(dithering)             \
2102 .align 3;                                                                      \
2103                                                                                \
2104 function(setup_blocks_shaded_untextured_##dithering##_unswizzled_indirect)     \
2105   ldrh num_spans, [ psx_gpu, #psx_gpu_num_spans_offset ];                      \
2106   add rg_dx_ptr, psx_gpu, #(psx_gpu_uvrg_dx_offset + 8);                       \
2107                                                                                \
2108   vld1.u32 { rg_dx }, [ rg_dx_ptr, :64 ];                                      \
2109                                                                                \
2110   cmp num_spans, #0;                                                           \
2111   bxeq lr;                                                                     \
2112                                                                                \
2113   stmdb sp!, { r4 - r11, r14 };                                                \
2114   vshl.u32 rg_dx4, rg_dx, #2;                                                  \
2115                                                                                \
2116   ldr b_dx, [ psx_gpu, #psx_gpu_b_dx_offset ];                                 \
2117   vshl.u32 rg_dx8, rg_dx, #3;                                                  \
2118                                                                                \
2119   add span_uvrg_offset, psx_gpu, #psx_gpu_span_uvrg_offset_offset;             \
2120                                                                                \
2121   ldrh num_blocks, [ psx_gpu, #psx_gpu_num_blocks_offset ];                    \
2122   add span_edge_data, psx_gpu, #psx_gpu_span_edge_data_offset;                 \
2123                                                                                \
2124   add span_b_offset, psx_gpu, #psx_gpu_span_b_offset_offset;                   \
2125   add block_ptr_a, psx_gpu, #psx_gpu_blocks_offset;                            \
2126                                                                                \
2127   add block_ptr_a, block_ptr_a, num_blocks, lsl #6;                            \
2128   vmov.u8 d64_1, #1;                                                           \
2129                                                                                \
2130   vmov.u8 d128_4, #4;                                                          \
2131   vmov.u8 d64_128, #128;                                                       \
2132                                                                                \
2133   vmov.u8 d128_0x7, #0x7;                                                      \
2134                                                                                \
2135  0:                                                                            \
2136   ldrh span_num_blocks, [ span_edge_data, #edge_data_num_blocks_offset ];      \
2137   add dither_offset_ptr, psx_gpu, #psx_gpu_dither_table_offset;                \
2138                                                                                \
2139   ldrh y, [ span_edge_data, #edge_data_y_offset ];                             \
2140   ldr fb_ptr, [ psx_gpu, #psx_gpu_vram_ptr_offset ];                           \
2141                                                                                \
2142   cmp span_num_blocks, #0;                                                     \
2143   beq 1f;                                                                      \
2144                                                                                \
2145   ldrh left_x, [ span_edge_data, #edge_data_left_x_offset ];                   \
2146   add num_blocks, span_num_blocks, num_blocks;                                 \
2147                                                                                \
2148   cmp num_blocks, #MAX_BLOCKS;                                                 \
2149   bgt 2f;                                                                      \
2150                                                                                \
2151  3:                                                                            \
2152   ldr b, [ span_b_offset ];                                                    \
2153   add fb_ptr, fb_ptr, y, lsl #11;                                              \
2154                                                                                \
2155   vdup.u32 v_left_x, left_x;                                                   \
2156   and y, y, #0x3;                                                              \
2157                                                                                \
2158   ldr dither_row, [ dither_offset_ptr, y, lsl #2 ];                            \
2159   add fb_ptr, fb_ptr, left_x, lsl #1;                                          \
2160                                                                                \
2161   mla b, b_dx, left_x, b;                                                      \
2162   and dither_shift, left_x, #0x03;                                             \
2163                                                                                \
2164   vld1.u32 { uvrg }, [ span_uvrg_offset, :128 ];                               \
2165   vshr.u32 rg_dx, rg_dx4, #2;                                                  \
2166                                                                                \
2167   mov dither_shift, dither_shift, lsl #3;                                      \
2168   vmla.u32 rg, rg_dx, v_left_x;                                                \
2169                                                                                \
2170   mov c_64, #64;                                                               \
2171   subs span_num_blocks, span_num_blocks, #1;                                   \
2172                                                                                \
2173   mov dither_row, dither_row, ror dither_shift;                                \
2174   mov b_dx4, b_dx, lsl #2;                                                     \
2175                                                                                \
2176   vdup.u32 dither_offsets, dither_row;                                         \
2177   add block_span_ptr, psx_gpu, #psx_gpu_r_block_span_offset;                   \
2178                                                                                \
2179   vdup.u32 b_block, b;                                                         \
2180   vadd.u8 dither_offsets, dither_offsets, d128_4;                              \
2181                                                                                \
2182   mov b_dx8, b_dx, lsl #3;                                                     \
2183   vdup.u32 r_block, rg[0];                                                     \
2184   vdup.u32 g_block, rg[1];                                                     \
2185                                                                                \
2186   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2187                                                                                \
2188   vadd.u32 r_block, r_block, block_span;                                       \
2189   vld1.u32 { block_span }, [ block_span_ptr, :128 ]!;                          \
2190                                                                                \
2191   vadd.u32 g_block, g_block, block_span;                                       \
2192   vld1.u32 { block_span }, [ block_span_ptr, :128 ];                           \
2193                                                                                \
2194   vadd.u32 b_block, b_block, block_span;                                       \
2195   add block_ptr_b, block_ptr_a, #16;                                           \
2196                                                                                \
2197   vshrn.u32 r_whole_low, r_block, #16;                                         \
2198   vshrn.u32 g_whole_low, g_block, #16;                                         \
2199   vshrn.u32 b_whole_low, b_block, #16;                                         \
2200   vdup.u32 dx4, rg_dx4[0];                                                     \
2201                                                                                \
2202   vaddhn.u32 r_whole_high, r_block, dx4;                                       \
2203   vdup.u32 dx4, rg_dx4[1];                                                     \
2204                                                                                \
2205   vaddhn.u32 g_whole_high, g_block, dx4;                                       \
2206   vdup.u32 dx4, b_dx4;                                                         \
2207                                                                                \
2208   vaddhn.u32 b_whole_high, b_block, dx4;                                       \
2209   vdup.u32 dx8, rg_dx8[0];                                                     \
2210                                                                                \
2211   vadd.u32 r_block, r_block, dx8;                                              \
2212   vdup.u32 dx8, rg_dx8[1];                                                     \
2213                                                                                \
2214   vadd.u32 g_block, g_block, dx8;                                              \
2215   vdup.u32 dx8, b_dx8;                                                         \
2216                                                                                \
2217   vadd.u32 b_block, b_block, dx8;                                              \
2218                                                                                \
2219   vmovn.u16 r_whole_8, r_whole;                                                \
2220   vmovn.u16 g_whole_8, g_whole;                                                \
2221   vmovn.u16 b_whole_8, b_whole;                                                \
2222                                                                                \
2223   beq 5f;                                                                      \
2224   veor.u32 draw_mask, draw_mask, draw_mask;                                    \
2225                                                                                \
2226  4:                                                                            \
2227   setup_blocks_shaded_untextured_dither_a_##dithering();                       \
2228   vshrn.u32 r_whole_low, r_block, #16;                                         \
2229                                                                                \
2230   setup_blocks_shaded_untextured_dither_b_##dithering();                       \
2231   vshrn.u32 g_whole_low, g_block, #16;                                         \
2232                                                                                \
2233   vshrn.u32 b_whole_low, b_block, #16;                                         \
2234   str fb_ptr, [ block_ptr_a, #44 ];      &n