plugins/gpu_neon/psx_gpu/psx_gpu.c

   1 /*
   2  * Copyright (C) 2011 Gilead Kutnick "Exophase" <exophase@gmail.com>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of
   7  * the License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  */
  14
  15 #include <stdio.h>
  16 #include <stdlib.h>
  17 #include <stdint.h>
  18 #include <string.h>
  19 #include <assert.h>
  20
  21 #include "common.h"
  22 #ifndef NEON_BUILD
  23 #include "vector_ops.h"
  24 #endif
  25 #include "psx_gpu_simd.h"
  26
  27 #if 0
  28 void dump_r_d(const char *name, void *dump);
  29 void dump_r_q(const char *name, void *dump);
  30 #define dumprd(n) dump_r_d(#n, n.e)
  31 #define dumprq(n) dump_r_q(#n, n.e)
  32 #endif
  33
  34 u32 span_pixels = 0;
  35 u32 span_pixel_blocks = 0;
  36 u32 spans = 0;
  37 u32 triangles = 0;
  38 u32 sprites = 0;
  39 u32 sprites_4bpp = 0;
  40 u32 sprites_8bpp = 0;
  41 u32 sprites_16bpp = 0;
  42 u32 sprite_blocks = 0;
  43 u32 sprites_untextured = 0;
  44 u32 lines = 0;
  45 u32 trivial_rejects = 0;
  46 u32 texels_4bpp = 0;
  47 u32 texels_8bpp = 0;
  48 u32 texels_16bpp = 0;
  49 u32 texel_blocks_4bpp = 0;
  50 u32 texel_blocks_8bpp = 0;
  51 u32 texel_blocks_16bpp = 0;
  52 u32 texel_blocks_untextured = 0;
  53 u32 blend_blocks = 0;
  54 u32 render_buffer_flushes = 0;
  55 u32 state_changes = 0;
  56 u32 left_split_triangles = 0;
  57 u32 flat_triangles = 0;
  58 u32 clipped_triangles = 0;
  59 u32 zero_block_spans = 0;
  60 u32 texture_cache_loads = 0;
  61 u32 false_modulated_blocks = 0;
  62
  63 #define stats_add(stat, count) // stat += count
  64
  65 /* double size for enhancement */
  66 u32 reciprocal_table[512 * 2];
  67
  68
  69 typedef s32 fixed_type;
  70
  71 #define EDGE_STEP_BITS 32
  72 #define FIXED_BITS     12
  73
  74 #define fixed_center(value)                                                    \
  75   ((((fixed_type)(value)) << FIXED_BITS) + (1 << (FIXED_BITS - 1)))            \
  76
  77 #define int_to_fixed(value)                                                    \
  78   (((fixed_type)(value)) << FIXED_BITS)                                        \
  79
  80 #define fixed_to_int(value)                                                    \
  81   ((value) >> FIXED_BITS)                                                      \
  82
  83 #define fixed_to_double(value)                                                 \
  84   ((value) / (double)(1 << FIXED_BITS))                                        \
  85
  86 #define double_to_fixed(value)                                                 \
  87   (fixed_type)(((value) * (double)(1 << FIXED_BITS)))                          \
  88
  89 typedef void (setup_blocks_function_type)(psx_gpu_struct *psx_gpu);
  90 typedef void (texture_blocks_function_type)(psx_gpu_struct *psx_gpu);
  91 typedef void (shade_blocks_function_type)(psx_gpu_struct *psx_gpu);
  92 typedef void (blend_blocks_function_type)(psx_gpu_struct *psx_gpu);
  93
  94 typedef void (setup_sprite_function_type)(psx_gpu_struct *psx_gpu, s32 x,
  95  s32 y, s32 u, s32 v, s32 width, s32 height, u32 color);
  96
  97 struct render_block_handler_struct
  98 {
  99   void *setup_blocks;
 100   texture_blocks_function_type *texture_blocks;
 101   shade_blocks_function_type *shade_blocks;
 102   blend_blocks_function_type *blend_blocks;
 103 };
 104
 105 #ifndef NEON_BUILD
 106
 107 u32 fixed_reciprocal(u32 denominator, u32 *_shift)
 108 {
 109   u32 shift = __builtin_clz(denominator);
 110   u32 denominator_normalized = denominator << shift;
 111
 112   double numerator = (1ULL << 62) + denominator_normalized;
 113   double numerator_b;
 114
 115   double denominator_normalized_dp_b;
 116   u64 denominator_normalized_dp_u64;
 117
 118   u32 reciprocal;
 119   double reciprocal_dp;
 120
 121   u64 numerator_u64 = (denominator_normalized >> 10) |
 122    ((u64)(62 + 1023) << 52);
 123   *((u64 *)(&numerator_b)) = numerator_u64;
 124
 125   denominator_normalized_dp_u64 =
 126    (u64)(denominator_normalized << 21) |
 127    ((u64)((denominator_normalized >> 11) + ((1022 + 31) << 20)) << 32);
 128   *((u64 *)(&denominator_normalized_dp_b)) = denominator_normalized_dp_u64;
 129
 130   // Implement with a DP divide
 131   reciprocal_dp = numerator / denominator_normalized_dp_b;
 132   reciprocal = reciprocal_dp;
 133
 134   if(reciprocal == 0x80000001)
 135     reciprocal = 0x80000000;
 136
 137   *_shift = 62 - shift;
 138   return reciprocal;
 139 }
 140
 141 double reciprocal_estimate(double a)
 142 {
 143   int q, s;
 144   double r;
 145
 146   q = (int)(a * 512.0);
 147   /* a in units of 1/512 rounded down */
 148   r = 1.0 / (((double)q + 0.5) / 512.0); /* reciprocal r */
 149   s = (int)(256.0 * r + 0.5);
 150
 151   /* r in units of 1/256 rounded to nearest */
 152
 153   return (double)s / 256.0;
 154 }
 155
 156 u32 reciprocal_estimate_u32(u32 value)
 157 {
 158   u64 dp_value_u64;
 159   volatile double dp_value;
 160   volatile u64 *dp_value_ptr = (volatile u64 *)&dp_value;
 161
 162   if((value >> 31) == 0)
 163     return 0xFFFFFFFF;
 164
 165   dp_value_u64 = (0x3FEULL << (31 + 21)) | ((u64)(value & 0x7FFFFFFF) << 21);
 166
 167   *dp_value_ptr = dp_value_u64;
 168
 169   dp_value = reciprocal_estimate(dp_value);
 170   dp_value_u64 = *dp_value_ptr;
 171
 172   return (0x80000000 | ((dp_value_u64 >> 21) & 0x7FFFFFFF));
 173 }
 174
 175 u32 fixed_reciprocal_nr(u32 value, u32 *_shift)
 176 {
 177   u32 shift = __builtin_clz(value);
 178   u32 value_normalized = value << shift;
 179
 180   *_shift = 62 - shift;
 181
 182   value_normalized -= 2;
 183
 184   u32 reciprocal_normalized = reciprocal_estimate_u32(value_normalized) >> 1;
 185
 186   u32 temp = -(((u64)value_normalized * (u32)reciprocal_normalized) >> 31);
 187   reciprocal_normalized = (((u64)reciprocal_normalized * temp) >> 31);
 188   temp = -(((u64)value_normalized * (u32)reciprocal_normalized) >> 31);
 189   reciprocal_normalized = (((u64)reciprocal_normalized * temp) >> 31);
 190   temp = -(((u64)value_normalized * (u32)reciprocal_normalized) >> 31);
 191   reciprocal_normalized = (((u64)reciprocal_normalized * temp) >> 31);
 192
 193   return reciprocal_normalized;
 194 }
 195
 196 #endif
 197
 198
 199 s32 triangle_signed_area_x2(s32 x0, s32 y0, s32 x1, s32 y1, s32 x2, s32 y2)
 200 {
 201         return ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0));
 202 }
 203
 204 u32 texture_region_mask(s32 x1, s32 y1, s32 x2, s32 y2)
 205 {
 206   s32 coverage_x, coverage_y;
 207
 208   u32 mask_up_left;
 209   u32 mask_down_right;
 210
 211   coverage_x = x2 >> 6;
 212   coverage_y = y2 >> 8;
 213
 214   if(coverage_x < 0)
 215     coverage_x = 0;
 216
 217   if(coverage_x > 31)
 218     coverage_x = 31;
 219
 220   mask_down_right = ~(0xFFFFFFFF << (coverage_x + 1)) & 0xFFFF;
 221
 222   if(coverage_y >= 1)
 223     mask_down_right |= mask_down_right << 16;
 224
 225   coverage_x = x1 >> 6;
 226
 227   mask_up_left = 0xFFFF0000 << coverage_x;
 228   if(coverage_x < 0)
 229     mask_up_left = 0xFFFF0000;
 230
 231   coverage_y = y1 >> 8;
 232   if(coverage_y <= 0)
 233     mask_up_left |= mask_up_left >> 16;
 234
 235   return mask_up_left & mask_down_right;
 236 }
 237
 238 u32 invalidate_texture_cache_region(psx_gpu_struct *psx_gpu, u32 x1, u32 y1,
 239  u32 x2, u32 y2)
 240 {
 241   u32 mask = texture_region_mask(x1, y1, x2, y2);
 242
 243   psx_gpu->dirty_textures_4bpp_mask |= mask;
 244   psx_gpu->dirty_textures_8bpp_mask |= mask;
 245   psx_gpu->dirty_textures_8bpp_alternate_mask |= mask;
 246
 247   return mask;
 248 }
 249
 250 u32 invalidate_texture_cache_region_viewport(psx_gpu_struct *psx_gpu, u32 x1,
 251  u32 y1, u32 x2, u32 y2)
 252 {
 253   u32 mask = texture_region_mask(x1, y1, x2, y2) &
 254    psx_gpu->viewport_mask;
 255
 256   psx_gpu->dirty_textures_4bpp_mask |= mask;
 257   psx_gpu->dirty_textures_8bpp_mask |= mask;
 258   psx_gpu->dirty_textures_8bpp_alternate_mask |= mask;
 259
 260   return mask;
 261 }
 262
 263 static void update_texture_cache_region_(psx_gpu_struct *psx_gpu,
 264  u32 x1, u32 y1, u32 x2, u32 y2)
 265 {
 266   u32 mask = texture_region_mask(x1, y1, x2, y2);
 267   u32 texture_page;
 268   u8 *texture_page_ptr;
 269   u16 *vram_ptr;
 270   u32 texel_block;
 271   u32 sub_x, sub_y;
 272
 273   psx_gpu->dirty_textures_8bpp_mask |= mask;
 274   psx_gpu->dirty_textures_8bpp_alternate_mask |= mask;
 275
 276   if ((psx_gpu->dirty_textures_4bpp_mask & mask) == 0 &&
 277       (x1 & 3) == 0 && (y1 & 15) == 0 && x2 - x1 < 4 && y2 - y1 < 16)
 278   {
 279     texture_page = ((x1 / 64) & 15) + (y1 / 256) * 16;
 280     texture_page_ptr = psx_gpu->texture_4bpp_cache[texture_page];
 281     texture_page_ptr += (x1 / 4 & 15) * 16*16 + (y1 / 16 & 15) * 16*16*16;
 282     vram_ptr = psx_gpu->vram_ptr + x1 + y1 * 1024;
 283     sub_x = 4;
 284     sub_y = 16;
 285
 286     while(sub_y)
 287     {
 288       while(sub_x)
 289       {
 290         texel_block = *vram_ptr;
 291
 292         texture_page_ptr[0] = texel_block & 0xF;
 293         texture_page_ptr[1] = (texel_block >> 4) & 0xF;
 294         texture_page_ptr[2] = (texel_block >> 8) & 0xF;
 295         texture_page_ptr[3] = texel_block >> 12;
 296
 297         vram_ptr++;
 298         texture_page_ptr += 4;
 299
 300         sub_x--;
 301       }
 302
 303       vram_ptr -= 4;
 304       sub_x = 4;
 305
 306       sub_y--;
 307       vram_ptr += 1024;
 308     }
 309   }
 310   else
 311   {
 312     psx_gpu->dirty_textures_4bpp_mask |= mask;
 313   }
 314 }
 315
 316 void update_texture_cache_region(psx_gpu_struct *psx_gpu, u32 x1, u32 y1,
 317  u32 x2, u32 y2)
 318 {
 319   s32 w = x2 - x1;
 320   do
 321   {
 322     x2 = x1 + w;
 323     if (x2 > 1023)
 324       x2 = 1023;
 325     update_texture_cache_region_(psx_gpu, x1, y1, x2, y2);
 326     w -= x2 - x1;
 327     x1 = 0;
 328   }
 329   while (unlikely(w > 0));
 330 }
 331
 332 #ifndef NEON_BUILD
 333
 334 void update_texture_4bpp_cache(psx_gpu_struct *psx_gpu)
 335 {
 336   u32 current_texture_page = psx_gpu->current_texture_page;
 337   u8 *texture_page_ptr = psx_gpu->texture_page_base;
 338   u16 *vram_ptr = psx_gpu->vram_ptr;
 339
 340   u32 texel_block;
 341   u32 tile_x, tile_y;
 342   u32 sub_x, sub_y;
 343
 344   vram_ptr += (current_texture_page >> 4) * 256 * 1024;
 345   vram_ptr += (current_texture_page & 0xF) * 64;
 346
 347   texture_cache_loads++;
 348
 349   tile_y = 16;
 350   tile_x = 16;
 351   sub_x = 4;
 352   sub_y = 16;
 353
 354   psx_gpu->dirty_textures_4bpp_mask &= ~(psx_gpu->current_texture_mask);
 355
 356   while(tile_y)
 357   {
 358     while(tile_x)
 359     {
 360       while(sub_y)
 361       {
 362         while(sub_x)
 363         {
 364           texel_block = *vram_ptr;
 365
 366           texture_page_ptr[0] = texel_block & 0xF;
 367           texture_page_ptr[1] = (texel_block >> 4) & 0xF;
 368           texture_page_ptr[2] = (texel_block >> 8) & 0xF;
 369           texture_page_ptr[3] = texel_block >> 12;
 370
 371           vram_ptr++;
 372           texture_page_ptr += 4;
 373
 374           sub_x--;
 375         }
 376
 377         vram_ptr -= 4;
 378         sub_x = 4;
 379
 380         sub_y--;
 381         vram_ptr += 1024;
 382       }
 383
 384       sub_y = 16;
 385
 386       vram_ptr -= (1024 * 16) - 4;
 387       tile_x--;
 388     }
 389
 390     tile_x = 16;
 391
 392     vram_ptr += (16 * 1024) - (4 * 16);
 393     tile_y--;
 394   }
 395 }
 396
 397 void update_texture_8bpp_cache_slice(psx_gpu_struct *psx_gpu,
 398  u32 texture_page)
 399 {
 400   u16 *texture_page_ptr = psx_gpu->texture_page_base;
 401   u16 *vram_ptr = psx_gpu->vram_ptr;
 402
 403   u32 tile_x, tile_y;
 404   u32 sub_y;
 405
 406   vec_8x16u texels;
 407
 408   texture_cache_loads++;
 409
 410   vram_ptr += (texture_page >> 4) * 256 * 1024;
 411   vram_ptr += (texture_page & 0xF) * 64;
 412
 413   if((texture_page ^ psx_gpu->current_texture_page) & 0x1)
 414     texture_page_ptr += (8 * 16) * 8;
 415
 416   tile_x = 8;
 417   tile_y = 16;
 418
 419   sub_y = 16;
 420
 421   while(tile_y)
 422   {
 423     while(tile_x)
 424     {
 425       while(sub_y)
 426       {
 427         load_128b(texels, vram_ptr);
 428         store_128b(texels, texture_page_ptr);
 429
 430         texture_page_ptr += 8;
 431         vram_ptr += 1024;
 432
 433         sub_y--;
 434       }
 435
 436       sub_y = 16;
 437
 438       vram_ptr -= (1024 * 16);
 439       vram_ptr += 8;
 440
 441       tile_x--;
 442     }
 443
 444     tile_x = 8;
 445
 446     vram_ptr -= (8 * 8);
 447     vram_ptr += (16 * 1024);
 448
 449     texture_page_ptr += (8 * 16) * 8;
 450     tile_y--;
 451   }
 452 }
 453
 454 #endif
 455
 456
 457 void update_texture_8bpp_cache(psx_gpu_struct *psx_gpu)
 458 {
 459   u32 current_texture_page = psx_gpu->current_texture_page;
 460   u32 update_textures =
 461    psx_gpu->dirty_textures_8bpp_mask & psx_gpu->current_texture_mask;
 462
 463   psx_gpu->dirty_textures_8bpp_mask &= ~update_textures;
 464
 465   if(update_textures & (1 << current_texture_page))
 466   {
 467     update_texture_8bpp_cache_slice(psx_gpu, current_texture_page);
 468     update_textures &= ~(1 << current_texture_page);
 469   }
 470
 471   if(update_textures)
 472   {
 473     u32 adjacent_texture_page = ((current_texture_page + 1) & 0xF) |
 474      (current_texture_page & 0x10);
 475
 476     update_texture_8bpp_cache_slice(psx_gpu, adjacent_texture_page);
 477   }
 478 }
 479
 480 void flush_render_block_buffer(psx_gpu_struct *psx_gpu)
 481 {
 482   if((psx_gpu->render_mode & RENDER_INTERLACE_ENABLED) &&
 483    (psx_gpu->primitive_type == PRIMITIVE_TYPE_SPRITE))
 484   {
 485     u32 num_blocks_dest = 0;
 486     block_struct *block_src = psx_gpu->blocks;
 487     block_struct *block_dest = psx_gpu->blocks;
 488
 489     u16 *vram_ptr = psx_gpu->vram_ptr;
 490     u32 i;
 491
 492     if(psx_gpu->render_mode & RENDER_INTERLACE_ODD)
 493     {
 494       for(i = 0; i < psx_gpu->num_blocks; i++)
 495       {
 496         u32 fb_offset = (u32)((u8 *)block_src->fb_ptr - (u8 *)vram_ptr);
 497         if(fb_offset & (1 << 11))
 498         {
 499           *block_dest = *block_src;
 500           num_blocks_dest++;
 501           block_dest++;
 502         }
 503         block_src++;
 504       }
 505     }
 506     else
 507     {
 508       for(i = 0; i < psx_gpu->num_blocks; i++)
 509       {
 510         u32 fb_offset = (u32)((u8 *)block_src->fb_ptr - (u8 *)vram_ptr);
 511         if((fb_offset & (1 << 11)) == 0)
 512         {
 513           *block_dest = *block_src;
 514           num_blocks_dest++;
 515           block_dest++;
 516         }
 517         block_src++;
 518       }
 519     }
 520
 521     psx_gpu->num_blocks = num_blocks_dest;
 522   }
 523
 524   if(psx_gpu->num_blocks)
 525   {
 526     render_block_handler_struct *render_block_handler =
 527      psx_gpu->render_block_handler;
 528
 529     render_block_handler->texture_blocks(psx_gpu);
 530     render_block_handler->shade_blocks(psx_gpu);
 531     render_block_handler->blend_blocks(psx_gpu);
 532
 533 #ifdef PROFILE
 534     span_pixel_blocks += psx_gpu->num_blocks;
 535     render_buffer_flushes++;
 536 #endif
 537
 538     psx_gpu->num_blocks = 0;
 539   }
 540 }
 541
 542
 543 #ifndef NEON_BUILD
 544
 545 #define setup_gradient_calculation_input(set, vertex)                          \
 546   /* First type is:  uvrg bxxx xxxx                                          */\
 547   /* Second type is: yyyy ybyy uvrg                                          */\
 548   /* Since x_a and y_c are the same the same variable is used for both.      */\
 549   x##set##_a_y##set##_c.e[0] = vertex->u;                                      \
 550   x##set##_a_y##set##_c.e[1] = vertex->v;                                      \
 551   x##set##_a_y##set##_c.e[2] = vertex->r;                                      \
 552   x##set##_a_y##set##_c.e[3] = vertex->g;                                      \
 553   dup_4x16b(x##set##_b, vertex->x);                                            \
 554   dup_4x16b(x##set##_c, vertex->x);                                            \
 555   dup_4x16b(y##set##_a, vertex->y);                                            \
 556   dup_4x16b(y##set##_b, vertex->y);                                            \
 557   x##set##_b.e[0] = vertex->b;                                                 \
 558   y##set##_b.e[1] = vertex->b                                                  \
 559
 560
 561 void compute_all_gradients(psx_gpu_struct *psx_gpu, vertex_struct *a,
 562  vertex_struct *b, vertex_struct *c)
 563 {
 564   u32 triangle_area = psx_gpu->triangle_area;
 565   u32 winding_mask_scalar;
 566
 567   u32 triangle_area_shift;
 568   u64 triangle_area_reciprocal =
 569    fixed_reciprocal(triangle_area, &triangle_area_shift);
 570   triangle_area_shift = -(triangle_area_shift - FIXED_BITS);
 571
 572   // ((x1 - x0) * (y2 - y1)) - ((x2 - x1) * (y1 - y0)) =
 573   // ( d0       *  d1      ) - ( d2       *  d3      ) =
 574   // ( m0                  ) - ( m1                  ) = gradient
 575
 576   // This is split to do 12 elements at a time over three sets: a, b, and c.
 577   // Technically we only need to do 10 elements (uvrgb_x and uvrgb_y), so
 578   // two of the slots are unused.
 579
 580   // Inputs are all 16-bit signed. The m0/m1 results are 32-bit signed, as
 581   // is g.
 582
 583   vec_4x16s x0_a_y0_c, x0_b, x0_c;
 584   vec_4x16s y0_a, y0_b;
 585   vec_4x16s x1_a_y1_c, x1_b, x1_c;
 586   vec_4x16s y1_a, y1_b;
 587   vec_4x16s x2_a_y2_c, x2_b, x2_c;
 588   vec_4x16s y2_a, y2_b;
 589
 590   vec_4x32u uvrg_base;
 591   vec_4x32u b_base;
 592   vec_4x32u uvrgb_phase;
 593
 594   vec_4x16s d0_a_d3_c, d0_b, d0_c;
 595   vec_4x16s d1_a, d1_b, d1_c_d2_a;
 596   vec_4x16s d2_b, d2_c;
 597   vec_4x16s d3_a, d3_b;
 598
 599   vec_4x32s m0_a, m0_b, m0_c;
 600   vec_4x32s m1_a, m1_b, m1_c;
 601
 602   vec_4x32u gradient_area_a, gradient_area_c;
 603   vec_2x32u gradient_area_b;
 604
 605   vec_4x32u gradient_area_sign_a, gradient_area_sign_c;
 606   vec_2x32u gradient_area_sign_b;
 607   vec_4x32u winding_mask;
 608
 609   vec_2x64u gradient_wide_a0, gradient_wide_a1;
 610   vec_2x64u gradient_wide_c0, gradient_wide_c1;
 611   vec_2x64u gradient_wide_b;
 612
 613   vec_4x32u gradient_a, gradient_c;
 614   vec_2x32u gradient_b;
 615   vec_16x8s gradient_shift;
 616
 617   setup_gradient_calculation_input(0, a);
 618   setup_gradient_calculation_input(1, b);
 619   setup_gradient_calculation_input(2, c);
 620
 621   dup_4x32b(uvrgb_phase, psx_gpu->uvrgb_phase);
 622   shl_long_4x16b(uvrg_base, x0_a_y0_c, 16);
 623   shl_long_4x16b(b_base, x0_b, 16);
 624
 625   add_4x32b(uvrg_base, uvrg_base, uvrgb_phase);
 626   add_4x32b(b_base, b_base, uvrgb_phase);
 627
 628   // Can probably pair these, but it'll require careful register allocation
 629   sub_4x16b(d0_a_d3_c, x1_a_y1_c, x0_a_y0_c);
 630   sub_4x16b(d1_c_d2_a, x2_a_y2_c, x1_a_y1_c);
 631
 632   sub_4x16b(d0_b, x1_b, x0_b);
 633   sub_4x16b(d0_c, x1_c, x0_c);
 634
 635   sub_4x16b(d1_a, y2_a, y1_a);
 636   sub_4x16b(d1_b, y2_b, y1_b);
 637
 638   sub_4x16b(d2_b, x2_b, x1_b);
 639   sub_4x16b(d2_c, x2_c, x1_c);
 640
 641   sub_4x16b(d3_a, y1_a, y0_a);
 642   sub_4x16b(d3_b, y1_b, y0_b);
 643
 644   mul_long_4x16b(m0_a, d0_a_d3_c, d1_a);
 645   mul_long_4x16b(m0_b, d0_b, d1_b);
 646   mul_long_4x16b(m0_c, d0_c, d1_c_d2_a);
 647
 648   mul_long_4x16b(m1_a, d1_c_d2_a, d3_a);
 649   mul_long_4x16b(m1_b, d2_b, d3_b);
 650   mul_long_4x16b(m1_c, d2_c, d0_a_d3_c);
 651
 652   sub_4x32b(gradient_area_a, m0_a, m1_a);
 653   sub_2x32b(gradient_area_b, m0_b.low, m1_b.low);
 654   sub_4x32b(gradient_area_c, m0_c, m1_c);
 655
 656   cmpltz_4x32b(gradient_area_sign_a, gradient_area_a);
 657   cmpltz_2x32b(gradient_area_sign_b, gradient_area_b);
 658   cmpltz_4x32b(gradient_area_sign_c, gradient_area_c);
 659
 660   abs_4x32b(gradient_area_a, gradient_area_a);
 661   abs_2x32b(gradient_area_b, gradient_area_b);
 662   abs_4x32b(gradient_area_c, gradient_area_c);
 663
 664   winding_mask_scalar = -psx_gpu->triangle_winding;
 665
 666   dup_4x32b(winding_mask, winding_mask_scalar);
 667   eor_4x32b(gradient_area_sign_a, gradient_area_sign_a, winding_mask);
 668   eor_2x32b(gradient_area_sign_b, gradient_area_sign_b, winding_mask);
 669   eor_4x32b(gradient_area_sign_c, gradient_area_sign_c, winding_mask);
 670
 671   mul_scalar_long_2x32b(gradient_wide_a0,
 672    vector_cast(vec_2x32s, gradient_area_a.low),
 673    (s64)triangle_area_reciprocal);
 674   mul_scalar_long_2x32b(gradient_wide_a1,
 675    vector_cast(vec_2x32s, gradient_area_a.high),
 676    (s64)triangle_area_reciprocal);
 677   mul_scalar_long_2x32b(gradient_wide_b,
 678    vector_cast(vec_2x32s, gradient_area_b),
 679    (s64)triangle_area_reciprocal);
 680   mul_scalar_long_2x32b(gradient_wide_c0,
 681    vector_cast(vec_2x32s, gradient_area_c.low),
 682    (s64)triangle_area_reciprocal);
 683   mul_scalar_long_2x32b(gradient_wide_c1,
 684    vector_cast(vec_2x32s, gradient_area_c.high),
 685    (s64)triangle_area_reciprocal);
 686
 687   dup_16x8b(gradient_shift, triangle_area_shift);
 688   shl_reg_2x64b(gradient_wide_a0, gradient_wide_a0,
 689    vector_cast(vec_2x64u, gradient_shift));
 690   shl_reg_2x64b(gradient_wide_a1, gradient_wide_a1,
 691    vector_cast(vec_2x64u, gradient_shift));
 692   shl_reg_2x64b(gradient_wide_b, gradient_wide_b,
 693    vector_cast(vec_2x64u, gradient_shift));
 694   shl_reg_2x64b(gradient_wide_c0, gradient_wide_c0,
 695    vector_cast(vec_2x64u, gradient_shift));
 696   shl_reg_2x64b(gradient_wide_c1, gradient_wide_c1,
 697    vector_cast(vec_2x64u, gradient_shift));
 698
 699   mov_narrow_2x64b(gradient_a.low, gradient_wide_a0);
 700   mov_narrow_2x64b(gradient_a.high, gradient_wide_a1);
 701   mov_narrow_2x64b(gradient_b, gradient_wide_b);
 702   mov_narrow_2x64b(gradient_c.low, gradient_wide_c0);
 703   mov_narrow_2x64b(gradient_c.high, gradient_wide_c1);
 704
 705   shl_4x32b(gradient_a, gradient_a, 4);
 706   shl_2x32b(gradient_b, gradient_b, 4);
 707   shl_4x32b(gradient_c, gradient_c, 4);
 708
 709   eor_4x32b(gradient_a, gradient_a, gradient_area_sign_a);
 710   eor_2x32b(gradient_b, gradient_b, gradient_area_sign_b);
 711   eor_4x32b(gradient_c, gradient_c, gradient_area_sign_c);
 712
 713   sub_4x32b(gradient_a, gradient_a, gradient_area_sign_a);
 714   sub_2x32b(gradient_b, gradient_b, gradient_area_sign_b);
 715   sub_4x32b(gradient_c, gradient_c, gradient_area_sign_c);
 716
 717   u32 left_adjust = a->x;
 718   mls_scalar_4x32b(uvrg_base, gradient_a, left_adjust);
 719   mls_scalar_2x32b(b_base.low, gradient_b, left_adjust);
 720
 721   vec_4x32u uvrg_dx2;
 722   vec_2x32u b_dx2;
 723
 724   vec_4x32u uvrg_dx3;
 725   vec_2x32u b_dx3;
 726
 727   vec_4x32u zero;
 728
 729   eor_4x32b(zero, zero, zero);
 730   add_4x32b(uvrg_dx2, gradient_a, gradient_a);
 731   add_2x32b(b_dx2, gradient_b, gradient_b);
 732   add_4x32b(uvrg_dx3, gradient_a, uvrg_dx2);
 733   add_2x32b(b_dx3, gradient_b, b_dx2);
 734
 735   // Can be done with vst4, assuming that the zero, dx, dx2, and dx3 are
 736   // lined up properly
 737   psx_gpu->u_block_span.e[0] = zero.e[0];
 738   psx_gpu->u_block_span.e[1] = gradient_a.e[0];
 739   psx_gpu->u_block_span.e[2] = uvrg_dx2.e[0];
 740   psx_gpu->u_block_span.e[3] = uvrg_dx3.e[0];
 741
 742   psx_gpu->v_block_span.e[0] = zero.e[1];
 743   psx_gpu->v_block_span.e[1] = gradient_a.e[1];
 744   psx_gpu->v_block_span.e[2] = uvrg_dx2.e[1];
 745   psx_gpu->v_block_span.e[3] = uvrg_dx3.e[1];
 746
 747   psx_gpu->r_block_span.e[0] = zero.e[2];
 748   psx_gpu->r_block_span.e[1] = gradient_a.e[2];
 749   psx_gpu->r_block_span.e[2] = uvrg_dx2.e[2];
 750   psx_gpu->r_block_span.e[3] = uvrg_dx3.e[2];
 751
 752   psx_gpu->g_block_span.e[0] = zero.e[3];
 753   psx_gpu->g_block_span.e[1] = gradient_a.e[3];
 754   psx_gpu->g_block_span.e[2] = uvrg_dx2.e[3];
 755   psx_gpu->g_block_span.e[3] = uvrg_dx3.e[3];
 756
 757   psx_gpu->b_block_span.e[0] = zero.e[0];
 758   psx_gpu->b_block_span.e[1] = gradient_b.e[0];
 759   psx_gpu->b_block_span.e[2] = b_dx2.e[0];
 760   psx_gpu->b_block_span.e[3] = b_dx3.e[0];
 761
 762   psx_gpu->uvrg = uvrg_base;
 763   psx_gpu->b = b_base.e[0];
 764
 765   psx_gpu->uvrg_dx = gradient_a;
 766   psx_gpu->uvrg_dy = gradient_c;
 767   psx_gpu->b_dy = gradient_b.e[1];
 768 }
 769 #endif
 770
 771 #define vector_check(_a, _b)                                                   \
 772   if(memcmp(&_a, &_b, sizeof(_b)))                                             \
 773   {                                                                            \
 774     if(sizeof(_b) == 8)                                                        \
 775     {                                                                          \
 776       printf("mismatch on %s vs %s: (%x %x) vs (%x %x)\n",                     \
 777        #_a, #_b, _a.e[0], _a.e[1], _b.e[0], _b.e[1]);                          \
 778     }                                                                          \
 779     else                                                                       \
 780     {                                                                          \
 781       printf("mismatch on %s vs %s: (%x %x %x %x) vs (%x %x %x %x)\n",         \
 782        #_a, #_b, _a.e[0], _a.e[1], _a.e[2], _a.e[3], _b.e[0], _b.e[1],         \
 783        _b.e[2], _b.e[3]);                                                      \
 784     }                                                                          \
 785   }                                                                            \
 786
 787 #define scalar_check(_a, _b)                                                   \
 788   if(_a != _b)                                                                 \
 789     printf("mismatch on %s %s: %x vs %x\n", #_a, #_b, _a, _b)                  \
 790
 791
 792 #if !defined(NEON_BUILD) && !defined(NDEBUG)
 793 static void setup_spans_debug_check(psx_gpu_struct *psx_gpu,
 794   edge_data_struct *span_edge_data_element)
 795 {
 796   u32 _num_spans = span_edge_data_element - psx_gpu->span_edge_data;
 797   if (_num_spans > MAX_SPANS)
 798     *(volatile int *)0 = 1;
 799   if (_num_spans < psx_gpu->num_spans)
 800   {
 801     if(span_edge_data_element->num_blocks > MAX_BLOCKS_PER_ROW)
 802       *(volatile int *)0 = 2;
 803     if(span_edge_data_element->y >= 2048)
 804       *(volatile int *)0 = 3;
 805   }
 806 }
 807 #else
 808 #define setup_spans_debug_check(psx_gpu, span_edge_data_element)
 809 #endif
 810
 811 #define setup_spans_prologue_alternate_yes()                                   \
 812   vec_2x64s alternate_x;                                                       \
 813   vec_2x64s alternate_dx_dy;                                                   \
 814   vec_4x32s alternate_x_32;                                                    \
 815   vec_4x16u alternate_x_16;                                                    \
 816                                                                                \
 817   vec_4x16u alternate_select;                                                  \
 818   vec_4x16s y_mid_point;                                                       \
 819                                                                                \
 820   s32 y_b = v_b->y;                                                            \
 821   s64 edge_alt;                                                                \
 822   s32 edge_dx_dy_alt;                                                          \
 823   u32 edge_shift_alt                                                           \
 824
 825 #define setup_spans_prologue_alternate_no()                                    \
 826
 827 #define setup_spans_prologue(alternate_active)                                 \
 828   edge_data_struct *span_edge_data;                                            \
 829   vec_4x32u *span_uvrg_offset;                                                 \
 830   u32 *span_b_offset;                                                          \
 831                                                                                \
 832   s32 clip;                                                                    \
 833                                                                                \
 834   vec_2x64s edges_xy;                                                          \
 835   vec_2x32s edges_dx_dy;                                                       \
 836   vec_2x32u edge_shifts;                                                       \
 837                                                                                \
 838   vec_2x64s left_x, right_x;                                                   \
 839   vec_2x64s left_dx_dy, right_dx_dy;                                           \
 840   vec_4x32s left_x_32, right_x_32;                                             \
 841   vec_8x16s left_right_x_16;                                                   \
 842   vec_4x16s y_x4;                                                              \
 843   vec_8x16s left_edge;                                                         \
 844   vec_8x16s right_edge;                                                        \
 845   vec_4x16u span_shift;                                                        \
 846                                                                                \
 847   vec_2x32u c_0x01;                                                            \
 848   vec_4x16u c_0x04;                                                            \
 849   vec_4x16u c_0xFFFE;                                                          \
 850   vec_4x16u c_0x07;                                                            \
 851                                                                                \
 852   vec_2x32s x_starts;                                                          \
 853   vec_2x32s x_ends;                                                            \
 854                                                                                \
 855   s32 x_a = v_a->x;                                                            \
 856   s32 x_b = v_b->x;                                                            \
 857   s32 x_c = v_c->x;                                                            \
 858   s32 y_a = v_a->y;                                                            \
 859   s32 y_c = v_c->y;                                                            \
 860                                                                                \
 861   vec_4x32u uvrg = psx_gpu->uvrg;                                              \
 862   vec_4x32u uvrg_dy = psx_gpu->uvrg_dy;                                        \
 863   u32 b = psx_gpu->b;                                                          \
 864   u32 b_dy = psx_gpu->b_dy;                                                    \
 865                                                                                \
 866   dup_2x32b(c_0x01, 0x01);                                                     \
 867   setup_spans_prologue_alternate_##alternate_active()                          \
 868
 869 #define setup_spans_prologue_b()                                               \
 870   span_edge_data = psx_gpu->span_edge_data;                                    \
 871   span_uvrg_offset = psx_gpu->span_uvrg_offset;                                \
 872   span_b_offset = psx_gpu->span_b_offset;                                      \
 873                                                                                \
 874   vec_8x16u c_0x0001;                                                          \
 875   vec_4x16u c_max_blocks_per_row;                                              \
 876                                                                                \
 877   dup_8x16b(c_0x0001, 0x0001);                                                 \
 878   dup_8x16b(left_edge, psx_gpu->viewport_start_x);                             \
 879   dup_8x16b(right_edge, psx_gpu->viewport_end_x);                              \
 880   add_8x16b(right_edge, right_edge, c_0x0001);                                 \
 881   dup_4x16b(c_0x04, 0x04);                                                     \
 882   dup_4x16b(c_0x07, 0x07);                                                     \
 883   dup_4x16b(c_0xFFFE, 0xFFFE);                                                 \
 884   dup_4x16b(c_max_blocks_per_row, MAX_BLOCKS_PER_ROW);                         \
 885
 886
 887 #define compute_edge_delta_x2()                                                \
 888 {                                                                              \
 889   vec_2x32s heights;                                                           \
 890   vec_2x32s height_reciprocals;                                                \
 891   vec_2x32s heights_b;                                                         \
 892   vec_4x32u widths;                                                            \
 893                                                                                \
 894   u32 edge_shift = reciprocal_table[height];                                   \
 895                                                                                \
 896   dup_2x32b(heights, height);                                                  \
 897   sub_2x32b(widths, x_ends, x_starts);                                         \
 898                                                                                \
 899   dup_2x32b(edge_shifts, edge_shift);                                          \
 900   sub_2x32b(heights_b, heights, c_0x01);                                       \
 901   shr_2x32b(height_reciprocals, edge_shifts, 10);                              \
 902                                                                                \
 903   mla_2x32b(heights_b, x_starts, heights);                                     \
 904   bic_immediate_4x16b(vector_cast(vec_4x16u, edge_shifts), 0xE0);              \
 905   mul_2x32b(edges_dx_dy, widths, height_reciprocals);                          \
 906   mul_long_2x32b(edges_xy, heights_b, height_reciprocals);                     \
 907 }                                                                              \
 908
 909 #define compute_edge_delta_x3(start_c, height_a, height_b)                     \
 910 {                                                                              \
 911   vec_2x32s heights;                                                           \
 912   vec_2x32s height_reciprocals;                                                \
 913   vec_2x32s heights_b;                                                         \
 914   vec_2x32u widths;                                                            \
 915                                                                                \
 916   u32 width_alt;                                                               \
 917   s32 height_b_alt;                                                            \
 918   u32 height_reciprocal_alt;                                                   \
 919                                                                                \
 920   heights.e[0] = height_a;                                                     \
 921   heights.e[1] = height_b;                                                     \
 922                                                                                \
 923   edge_shifts.e[0] = reciprocal_table[height_a];                               \
 924   edge_shifts.e[1] = reciprocal_table[height_b];                               \
 925   edge_shift_alt = reciprocal_table[height_minor_b];                           \
 926                                                                                \
 927   sub_2x32b(widths, x_ends, x_starts);                                         \
 928   width_alt = x_c - start_c;                                                   \
 929                                                                                \
 930   shr_2x32b(height_reciprocals, edge_shifts, 10);                              \
 931   height_reciprocal_alt = edge_shift_alt >> 10;                                \
 932                                                                                \
 933   bic_immediate_4x16b(vector_cast(vec_4x16u, edge_shifts), 0xE0);              \
 934   edge_shift_alt &= 0x1F;                                                      \
 935                                                                                \
 936   sub_2x32b(heights_b, heights, c_0x01);                                       \
 937   height_b_alt = height_minor_b - 1;                                           \
 938                                                                                \
 939   mla_2x32b(heights_b, x_starts, heights);                                     \
 940   height_b_alt += height_minor_b * start_c;                                    \
 941                                                                                \
 942   mul_long_2x32b(edges_xy, heights_b, height_reciprocals);                     \
 943   edge_alt = (s64)height_b_alt * height_reciprocal_alt;                        \
 944                                                                                \
 945   mul_2x32b(edges_dx_dy, widths, height_reciprocals);                          \
 946   edge_dx_dy_alt = width_alt * height_reciprocal_alt;                          \
 947 }                                                                              \
 948
 949
 950 #define setup_spans_adjust_y_up()                                              \
 951   sub_4x32b(y_x4, y_x4, c_0x04)                                                \
 952
 953 #define setup_spans_adjust_y_down()                                            \
 954   add_4x32b(y_x4, y_x4, c_0x04)                                                \
 955
 956 #define setup_spans_adjust_interpolants_up()                                   \
 957   sub_4x32b(uvrg, uvrg, uvrg_dy);                                              \
 958   b -= b_dy                                                                    \
 959
 960 #define setup_spans_adjust_interpolants_down()                                 \
 961   add_4x32b(uvrg, uvrg, uvrg_dy);                                              \
 962   b += b_dy                                                                    \
 963
 964
 965 #define setup_spans_clip_interpolants_increment()                              \
 966   mla_scalar_4x32b(uvrg, uvrg_dy, clip);                                       \
 967   b += b_dy * clip                                                             \
 968
 969 #define setup_spans_clip_interpolants_decrement()                              \
 970   mls_scalar_4x32b(uvrg, uvrg_dy, clip);                                       \
 971   b -= b_dy * clip                                                             \
 972
 973 #define setup_spans_clip_alternate_yes()                                       \
 974   edge_alt += edge_dx_dy_alt * (s64)(clip)                                     \
 975
 976 #define setup_spans_clip_alternate_no()                                        \
 977
 978 #define setup_spans_clip(direction, alternate_active)                          \
 979 {                                                                              \
 980   clipped_triangles++;                                                         \
 981   mla_scalar_long_2x32b(edges_xy, edges_dx_dy, (s64)clip);                     \
 982   setup_spans_clip_alternate_##alternate_active();                             \
 983   setup_spans_clip_interpolants_##direction();                                 \
 984 }                                                                              \
 985
 986
 987 #define setup_spans_adjust_edges_alternate_no(left_index, right_index)         \
 988 {                                                                              \
 989   vec_2x64u edge_shifts_64;                                                    \
 990   vec_2x64s edges_dx_dy_64;                                                    \
 991                                                                                \
 992   mov_wide_2x32b(edge_shifts_64, edge_shifts);                                 \
 993   shl_variable_2x64b(edges_xy, edges_xy, edge_shifts_64);                      \
 994                                                                                \
 995   mov_wide_2x32b(edges_dx_dy_64, edges_dx_dy);                                 \
 996   shl_variable_2x64b(edges_dx_dy_64, edges_dx_dy_64, edge_shifts_64);          \
 997                                                                                \
 998   left_x.e[0] = edges_xy.e[left_index];                                        \
 999   right_x.e[0] = edges_xy.e[right_index];                                      \
1000                                                                                \
1001   left_dx_dy.e[0] = edges_dx_dy_64.e[left_index];                              \
1002   left_dx_dy.e[1] = edges_dx_dy_64.e[left_index];                              \
1003   right_dx_dy.e[0] = edges_dx_dy_64.e[right_index];                            \
1004   right_dx_dy.e[1] = edges_dx_dy_64.e[right_index];                            \
1005                                                                                \
1006   add_1x64b(left_x.high, left_x.low, left_dx_dy.low);                          \
1007   add_1x64b(right_x.high, right_x.low, right_dx_dy.low);                       \
1008                                                                                \
1009   add_2x64b(left_dx_dy, left_dx_dy, left_dx_dy);                               \
1010   add_2x64b(right_dx_dy, right_dx_dy, right_dx_dy);                            \
1011 }                                                                              \
1012
1013 #define setup_spans_adjust_edges_alternate_yes(left_index, right_index)        \
1014 {                                                                              \
1015   setup_spans_adjust_edges_alternate_no(left_index, right_index);              \
1016   s64 edge_dx_dy_alt_64;                                                       \
1017                                                                                \
1018   dup_4x16b(y_mid_point, y_b);                                                 \
1019                                                                                \
1020   edge_alt <<= edge_shift_alt;                                                 \
1021   edge_dx_dy_alt_64 = (s64)edge_dx_dy_alt << edge_shift_alt;                   \
1022                                                                                \
1023   alternate_x.e[0] = edge_alt;                                                 \
1024   alternate_dx_dy.e[0] = edge_dx_dy_alt_64;                                    \
1025   alternate_dx_dy.e[1] = edge_dx_dy_alt_64;                                    \
1026                                                                                \
1027   add_1x64b(alternate_x.high, alternate_x.low, alternate_dx_dy.low);           \
1028   add_2x64b(alternate_dx_dy, alternate_dx_dy, alternate_dx_dy);                \
1029 }                                                                              \
1030
1031
1032 #define setup_spans_y_select_up()                                              \
1033   cmplt_4x16b(alternate_select, y_x4, y_mid_point)                             \
1034
1035 #define setup_spans_y_select_down()                                            \
1036   cmpgt_4x16b(alternate_select, y_x4, y_mid_point)                             \
1037
1038 #define setup_spans_y_select_alternate_yes(direction)                          \
1039   setup_spans_y_select_##direction()                                           \
1040
1041 #define setup_spans_y_select_alternate_no(direction)                           \
1042
1043 #define setup_spans_alternate_select_left()                                    \
1044   bit_4x16b(left_right_x_16.low, alternate_x_16, alternate_select)             \
1045
1046 #define setup_spans_alternate_select_right()                                   \
1047   bit_4x16b(left_right_x_16.high, alternate_x_16, alternate_select)            \
1048
1049 #define setup_spans_alternate_select_none()                                    \
1050
1051 #define setup_spans_increment_alternate_yes()                                  \
1052   shr_narrow_2x64b(alternate_x_32.low, alternate_x, 32);                       \
1053   add_2x64b(alternate_x, alternate_x, alternate_dx_dy);                        \
1054   shr_narrow_2x64b(alternate_x_32.high, alternate_x, 32);                      \
1055   add_2x64b(alternate_x, alternate_x, alternate_dx_dy);                        \
1056   mov_narrow_4x32b(alternate_x_16, alternate_x_32)                             \
1057
1058 #define setup_spans_increment_alternate_no()                                   \
1059
1060 #define setup_spans_set_x4(alternate, direction, alternate_active)             \
1061 {                                                                              \
1062   span_uvrg_offset[0] = uvrg;                                                  \
1063   span_b_offset[0] = b;                                                        \
1064   setup_spans_adjust_interpolants_##direction();                               \
1065                                                                                \
1066   span_uvrg_offset[1] = uvrg;                                                  \
1067   span_b_offset[1] = b;                                                        \
1068   setup_spans_adjust_interpolants_##direction();                               \
1069                                                                                \
1070   span_uvrg_offset[2] = uvrg;                                                  \
1071   span_b_offset[2] = b;                                                        \
1072   setup_spans_adjust_interpolants_##direction();                               \
1073                                                                                \
1074   span_uvrg_offset[3] = uvrg;                                                  \
1075   span_b_offset[3] = b;                                                        \
1076   setup_spans_adjust_interpolants_##direction();                               \
1077                                                                                \
1078   span_uvrg_offset += 4;                                                       \
1079   span_b_offset += 4;                                                          \
1080                                                                                \
1081   shr_narrow_2x64b(left_x_32.low, left_x, 32);                                 \
1082   shr_narrow_2x64b(right_x_32.low, right_x, 32);                               \
1083                                                                                \
1084   add_2x64b(left_x, left_x, left_dx_dy);                                       \
1085   add_2x64b(right_x, right_x, right_dx_dy);                                    \
1086                                                                                \
1087   shr_narrow_2x64b(left_x_32.high, left_x, 32);                                \
1088   shr_narrow_2x64b(right_x_32.high, right_x, 32);                              \
1089                                                                                \
1090   add_2x64b(left_x, left_x, left_dx_dy);                                       \
1091   add_2x64b(right_x, right_x, right_dx_dy);                                    \
1092                                                                                \
1093   mov_narrow_4x32b(left_right_x_16.low, left_x_32);                            \
1094   mov_narrow_4x32b(left_right_x_16.high, right_x_32);                          \
1095                                                                                \
1096   setup_spans_increment_alternate_##alternate_active();                        \
1097   setup_spans_y_select_alternate_##alternate_active(direction);                \
1098   setup_spans_alternate_select_##alternate();                                  \
1099                                                                                \
1100   max_8x16b(left_right_x_16, left_right_x_16, left_edge);                      \
1101   min_8x16b(left_right_x_16, left_right_x_16, right_edge);                     \
1102                                                                                \
1103   sub_4x16b(left_right_x_16.high, left_right_x_16.high, left_right_x_16.low);  \
1104   add_4x16b(left_right_x_16.high, left_right_x_16.high, c_0x07);               \
1105   and_4x16b(span_shift, left_right_x_16.high, c_0x07);                         \
1106   shl_variable_4x16b(span_shift, c_0xFFFE, span_shift);                        \
1107   shr_4x16b(left_right_x_16.high, left_right_x_16.high, 3);                    \
1108   min_4x16b(left_right_x_16.high, left_right_x_16.high, c_max_blocks_per_row); \
1109                                                                                \
1110   u32 i;                                                                       \
1111   for(i = 0; i < 4; i++)                                                       \
1112   {                                                                            \
1113     span_edge_data[i].left_x = left_right_x_16.low.e[i];                       \
1114     span_edge_data[i].num_blocks = left_right_x_16.high.e[i];                  \
1115     span_edge_data[i].right_mask = span_shift.e[i];                            \
1116     span_edge_data[i].y = y_x4.e[i];                                           \
1117     setup_spans_debug_check(psx_gpu, &span_edge_data[i]);                      \
1118   }                                                                            \
1119                                                                                \
1120   span_edge_data += 4;                                                         \
1121                                                                                \
1122   setup_spans_adjust_y_##direction();                                          \
1123 }                                                                              \
1124
1125
1126 #define setup_spans_alternate_adjust_yes()                                     \
1127   edge_alt -= edge_dx_dy_alt * (s64)height_minor_a                             \
1128
1129 #define setup_spans_alternate_adjust_no()                                      \
1130
1131
1132 #define setup_spans_down(left_index, right_index, alternate, alternate_active) \
1133   setup_spans_alternate_adjust_##alternate_active();                           \
1134   if(y_c > psx_gpu->viewport_end_y)                                            \
1135     height -= y_c - psx_gpu->viewport_end_y - 1;                               \
1136                                                                                \
1137   clip = psx_gpu->viewport_start_y - y_a;                                      \
1138   if(clip > 0)                                                                 \
1139   {                                                                            \
1140     height -= clip;                                                            \
1141     y_a += clip;                                                               \
1142     setup_spans_clip(increment, alternate_active);                             \
1143   }                                                                            \
1144                                                                                \
1145   setup_spans_prologue_b();                                                    \
1146                                                                                \
1147   if (height > 512)                                                            \
1148     height = 512;                                                              \
1149   if (height > 0)                                                              \
1150   {                                                                            \
1151     y_x4.e[0] = y_a;                                                           \
1152     y_x4.e[1] = y_a + 1;                                                       \
1153     y_x4.e[2] = y_a + 2;                                                       \
1154     y_x4.e[3] = y_a + 3;                                                       \
1155     setup_spans_adjust_edges_alternate_##alternate_active(left_index,          \
1156      right_index);                                                             \
1157                                                                                \
1158     psx_gpu->num_spans = height;                                               \
1159     do                                                                         \
1160     {                                                                          \
1161       setup_spans_set_x4(alternate, down, alternate_active);                   \
1162       height -= 4;                                                             \
1163     } while(height > 0);                                                       \
1164   }                                                                            \
1165
1166
1167 #define setup_spans_alternate_pre_increment_yes()                              \
1168   edge_alt += edge_dx_dy_alt                                                   \
1169
1170 #define setup_spans_alternate_pre_increment_no()                               \
1171
1172 #define setup_spans_up_decrement_height_yes()                                  \
1173   height--                                                                     \
1174
1175 #define setup_spans_up_decrement_height_no()                                   \
1176   {}                                                                           \
1177
1178 #define setup_spans_up(left_index, right_index, alternate, alternate_active)   \
1179   setup_spans_alternate_adjust_##alternate_active();                           \
1180   y_a--;                                                                       \
1181                                                                                \
1182   if(y_c < psx_gpu->viewport_start_y)                                          \
1183     height -= psx_gpu->viewport_start_y - y_c;                                 \
1184   else                                                                         \
1185     setup_spans_up_decrement_height_##alternate_active();                      \
1186                                                                                \
1187   clip = y_a - psx_gpu->viewport_end_y;                                        \
1188   if(clip > 0)                                                                 \
1189   {                                                                            \
1190     height -= clip;                                                            \
1191     y_a -= clip;                                                               \
1192     setup_spans_clip(decrement, alternate_active);                             \
1193   }                                                                            \
1194                                                                                \
1195   setup_spans_prologue_b();                                                    \
1196                                                                                \
1197   if (height > 512)                                                            \
1198     height = 512;                                                              \
1199   if (height > 0)                                                              \
1200   {                                                                            \
1201     y_x4.e[0] = y_a;                                                           \
1202     y_x4.e[1] = y_a - 1;                                                       \
1203     y_x4.e[2] = y_a - 2;                                                       \
1204     y_x4.e[3] = y_a - 3;                                                       \
1205     add_wide_2x32b(edges_xy, edges_xy, edges_dx_dy);                           \
1206     setup_spans_alternate_pre_increment_##alternate_active();                  \
1207     setup_spans_adjust_edges_alternate_##alternate_active(left_index,          \
1208      right_index);                                                             \
1209     setup_spans_adjust_interpolants_up();                                      \
1210                                                                                \
1211     psx_gpu->num_spans = height;                                               \
1212     while(height > 0)                                                          \
1213     {                                                                          \
1214       setup_spans_set_x4(alternate, up, alternate_active);                     \
1215       height -= 4;                                                             \
1216     }                                                                          \
1217   }                                                                            \
1218
1219 #define index_left  0
1220 #define index_right 1
1221
1222 #define setup_spans_up_up(minor, major)                                        \
1223   setup_spans_prologue(yes);                                                   \
1224   s32 height_minor_a = y_a - y_b;                                              \
1225   s32 height_minor_b = y_b - y_c;                                              \
1226   s32 height = y_a - y_c;                                                      \
1227                                                                                \
1228   dup_2x32b(x_starts, x_a);                                                    \
1229   x_ends.e[0] = x_c;                                                           \
1230   x_ends.e[1] = x_b;                                                           \
1231                                                                                \
1232   compute_edge_delta_x3(x_b, height, height_minor_a);                          \
1233   setup_spans_up(index_##major, index_##minor, minor, yes)                     \
1234
1235
1236 #ifndef NEON_BUILD
1237
1238 void setup_spans_up_left(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1239  vertex_struct *v_b, vertex_struct *v_c)
1240 {
1241   setup_spans_up_up(left, right);
1242 }
1243
1244 void setup_spans_up_right(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1245  vertex_struct *v_b, vertex_struct *v_c)
1246 {
1247   setup_spans_up_up(right, left);
1248 }
1249
1250 #define setup_spans_down_down(minor, major)                                    \
1251   setup_spans_prologue(yes);                                                   \
1252   s32 height_minor_a = y_b - y_a;                                              \
1253   s32 height_minor_b = y_c - y_b;                                              \
1254   s32 height = y_c - y_a;                                                      \
1255                                                                                \
1256   dup_2x32b(x_starts, x_a);                                                    \
1257   x_ends.e[0] = x_c;                                                           \
1258   x_ends.e[1] = x_b;                                                           \
1259                                                                                \
1260   compute_edge_delta_x3(x_b, height, height_minor_a);                          \
1261   setup_spans_down(index_##major, index_##minor, minor, yes)                   \
1262
1263 void setup_spans_down_left(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1264  vertex_struct *v_b, vertex_struct *v_c)
1265 {
1266   setup_spans_down_down(left, right);
1267 }
1268
1269 void setup_spans_down_right(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1270  vertex_struct *v_b, vertex_struct *v_c)
1271 {
1272   setup_spans_down_down(right, left);
1273 }
1274
1275 #define setup_spans_up_flat()                                                  \
1276   s32 height = y_a - y_c;                                                      \
1277                                                                                \
1278   flat_triangles++;                                                            \
1279   compute_edge_delta_x2();                                                     \
1280   setup_spans_up(index_left, index_right, none, no)                            \
1281
1282 void setup_spans_up_a(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1283  vertex_struct *v_b, vertex_struct *v_c)
1284 {
1285   setup_spans_prologue(no);
1286   x_starts.e[0] = x_a;
1287   x_starts.e[1] = x_b;
1288   dup_2x32b(x_ends, x_c);
1289
1290   setup_spans_up_flat();
1291 }
1292
1293 void setup_spans_up_b(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1294  vertex_struct *v_b, vertex_struct *v_c)
1295 {
1296   setup_spans_prologue(no);
1297   dup_2x32b(x_starts, x_a);
1298   x_ends.e[0] = x_b;
1299   x_ends.e[1] = x_c;
1300
1301   setup_spans_up_flat();
1302 }
1303
1304 #define setup_spans_down_flat()                                                \
1305   s32 height = y_c - y_a;                                                      \
1306                                                                                \
1307   flat_triangles++;                                                            \
1308   compute_edge_delta_x2();                                                     \
1309   setup_spans_down(index_left, index_right, none, no)                          \
1310
1311 void setup_spans_down_a(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1312  vertex_struct *v_b, vertex_struct *v_c)
1313 {
1314   setup_spans_prologue(no);
1315   x_starts.e[0] = x_a;
1316   x_starts.e[1] = x_b;
1317   dup_2x32b(x_ends, x_c);
1318
1319   setup_spans_down_flat();
1320 }
1321
1322 void setup_spans_down_b(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1323  vertex_struct *v_b, vertex_struct *v_c)
1324 {
1325   setup_spans_prologue(no);
1326   dup_2x32b(x_starts, x_a);
1327   x_ends.e[0] = x_b;
1328   x_ends.e[1] = x_c;
1329
1330   setup_spans_down_flat();
1331 }
1332
1333 void setup_spans_up_down(psx_gpu_struct *psx_gpu, vertex_struct *v_a,
1334  vertex_struct *v_b, vertex_struct *v_c)
1335 {
1336   setup_spans_prologue(no);
1337
1338   s32 y_b = v_b->y;
1339   s64 edge_alt;
1340   s32 edge_dx_dy_alt;
1341   u32 edge_shift_alt;
1342
1343   s32 middle_y = y_a;
1344   s32 height_minor_a = y_a - y_b;
1345   s32 height_minor_b = y_c - y_a;
1346   s32 height_major = y_c - y_b;
1347
1348   vec_2x64s edges_xy_b;
1349   vec_2x32s edges_dx_dy_b;
1350   vec_2x32u edge_shifts_b;
1351
1352   vec_2x32s height_increment;
1353
1354   x_starts.e[0] = x_a;
1355   x_starts.e[1] = x_c;
1356   dup_2x32b(x_ends, x_b);
1357
1358   compute_edge_delta_x3(x_a, height_minor_a, height_major);
1359
1360   height_increment.e[0] = 0;
1361   height_increment.e[1] = height_minor_b;
1362
1363   mla_long_2x32b(edges_xy, edges_dx_dy, height_increment);
1364
1365   edges_xy_b.e[0] = edge_alt;
1366   edges_xy_b.e[1] = edges_xy.e[1];
1367
1368   edge_shifts_b = edge_shifts;
1369   edge_shifts_b.e[0] = edge_shift_alt;
1370
1371   neg_2x32b(edges_dx_dy_b, edges_dx_dy);
1372   edges_dx_dy_b.e[0] = edge_dx_dy_alt;
1373
1374   y_a--;
1375
1376   if(y_b < psx_gpu->viewport_start_y)
1377     height_minor_a -= psx_gpu->viewport_start_y - y_b;
1378
1379   clip = y_a - psx_gpu->viewport_end_y;
1380   if(clip > 0)
1381   {
1382     height_minor_a -= clip;
1383     y_a -= clip;
1384     setup_spans_clip(decrement, no);
1385   }
1386
1387   setup_spans_prologue_b();
1388
1389   if (height_minor_a > 512)
1390     height_minor_a = 512;
1391   if (height_minor_a > 0)
1392   {
1393     y_x4.e[0] = y_a;
1394     y_x4.e[1] = y_a - 1;
1395     y_x4.e[2] = y_a - 2;
1396     y_x4.e[3] = y_a - 3;
1397     add_wide_2x32b(edges_xy, edges_xy, edges_dx_dy);
1398     setup_spans_adjust_edges_alternate_no(index_left, index_right);
1399     setup_spans_adjust_interpolants_up();
1400
1401     psx_gpu->num_spans = height_minor_a;
1402     while(height_minor_a > 0)
1403     {
1404       setup_spans_set_x4(none, up, no);
1405       height_minor_a -= 4;
1406     }
1407
1408     span_edge_data += height_minor_a;
1409     span_uvrg_offset += height_minor_a;
1410     span_b_offset += height_minor_a;
1411   }
1412
1413   edges_xy = edges_xy_b;
1414   edges_dx_dy = edges_dx_dy_b;
1415   edge_shifts = edge_shifts_b;
1416
1417   uvrg = psx_gpu->uvrg;
1418   b = psx_gpu->b;
1419
1420   y_a = middle_y;
1421
1422   if(y_c > psx_gpu->viewport_end_y)
1423     height_minor_b -= y_c - psx_gpu->viewport_end_y - 1;
1424
1425   clip = psx_gpu->viewport_start_y - y_a;
1426   if(clip > 0)
1427   {
1428     height_minor_b -= clip;
1429     y_a += clip;
1430     setup_spans_clip(increment, no);
1431   }
1432
1433   if (height_minor_b > 512)
1434     height_minor_b = 512;
1435   if (height_minor_b > 0)
1436   {
1437     y_x4.e[0] = y_a;
1438     y_x4.e[1] = y_a + 1;
1439     y_x4.e[2] = y_a + 2;
1440     y_x4.e[3] = y_a + 3;
1441     setup_spans_adjust_edges_alternate_no(index_left, index_right);
1442
1443     // FIXME: overflow corner case
1444     if(psx_gpu->num_spans + height_minor_b == MAX_SPANS)
1445       height_minor_b &= ~3;
1446
1447     psx_gpu->num_spans += height_minor_b;
1448     while(height_minor_b > 0)
1449     {
1450       setup_spans_set_x4(none, down, no);
1451       height_minor_b -= 4;
1452     }
1453   }
1454
1455   left_split_triangles++;
1456 }
1457
1458 #endif
1459
1460
1461 #define dither_table_entry_normal(value)                                       \
1462   (value)                                                                      \
1463
1464
1465 #define setup_blocks_load_msb_mask_indirect()                                  \
1466
1467 #define setup_blocks_load_msb_mask_direct()                                    \
1468   vec_8x16u msb_mask;                                                          \
1469   dup_8x16b(msb_mask, psx_gpu->mask_msb);                                      \
1470
1471
1472 #define setup_blocks_variables_shaded_textured(target)                         \
1473   vec_4x32u u_block;                                                           \
1474   vec_4x32u v_block;                                                           \
1475   vec_4x32u r_block;                                                           \
1476   vec_4x32u g_block;                                                           \
1477   vec_4x32u b_block;                                                           \
1478   vec_4x32u uvrg_dx = psx_gpu->uvrg_dx;                                        \
1479   vec_4x32u uvrg_dx4;                                                          \
1480   vec_4x32u uvrg_dx8;                                                          \
1481   vec_4x32u uvrg;                                                              \
1482   u32 b_dx = psx_gpu->b_block_span.e[1];                                       \
1483   u32 b_dx4 = b_dx << 2;                                                       \
1484   u32 b_dx8 = b_dx << 3;                                                       \
1485   u32 b;                                                                       \
1486                                                                                \
1487   vec_16x8u texture_mask;                                                      \
1488   shl_4x32b(uvrg_dx4, uvrg_dx, 2);                                             \
1489   shl_4x32b(uvrg_dx8, uvrg_dx, 3);                                             \
1490   dup_8x8b(texture_mask.low, psx_gpu->texture_mask_width);                     \
1491   dup_8x8b(texture_mask.high, psx_gpu->texture_mask_height)                    \
1492
1493 #define setup_blocks_variables_shaded_untextured(target)                       \
1494   vec_4x32u r_block;                                                           \
1495   vec_4x32u g_block;                                                           \
1496   vec_4x32u b_block;                                                           \
1497   vec_4x32u rgb_dx;                                                            \
1498   vec_4x32u rgb_dx4;                                                           \
1499   vec_4x32u rgb_dx8;                                                           \
1500   vec_4x32u rgb;                                                               \
1501                                                                                \
1502   vec_8x8u d64_0x07;                                                           \
1503   vec_8x8u d64_1;                                                              \
1504   vec_8x8u d64_4;                                                              \
1505   vec_8x8u d64_128;                                                            \
1506                                                                                \
1507   dup_8x8b(d64_0x07, 0x07);                                                    \
1508   dup_8x8b(d64_1, 1);                                                          \
1509   dup_8x8b(d64_4, 4);                                                          \
1510   dup_8x8b(d64_128, 128);                                                      \
1511                                                                                \
1512   rgb_dx.low = psx_gpu->uvrg_dx.high;                                          \
1513   rgb_dx.e[2] = psx_gpu->b_block_span.e[1];                                    \
1514   shl_4x32b(rgb_dx4, rgb_dx, 2);                                               \
1515   shl_4x32b(rgb_dx8, rgb_dx, 3)                                                \
1516
1517 #define setup_blocks_variables_unshaded_textured(target)                       \
1518   vec_4x32u u_block;                                                           \
1519   vec_4x32u v_block;                                                           \
1520   vec_2x32u uv_dx = psx_gpu->uvrg_dx.low;                                      \
1521   vec_2x32u uv_dx4;                                                            \
1522   vec_2x32u uv_dx8;                                                            \
1523   vec_2x32u uv = psx_gpu->uvrg.low;                                            \
1524                                                                                \
1525   vec_16x8u texture_mask;                                                      \
1526   shl_2x32b(uv_dx4, uv_dx, 2);                                                 \
1527   shl_2x32b(uv_dx8, uv_dx, 3);                                                 \
1528   dup_8x8b(texture_mask.low, psx_gpu->texture_mask_width);                     \
1529   dup_8x8b(texture_mask.high, psx_gpu->texture_mask_height)                    \
1530
1531
1532 #define setup_blocks_variables_unshaded_untextured_direct()                    \
1533   or_8x16b(colors, colors, msb_mask)                                           \
1534
1535 #define setup_blocks_variables_unshaded_untextured_indirect()                  \
1536
1537 #define setup_blocks_variables_unshaded_untextured(target)                     \
1538   u32 color = psx_gpu->triangle_color;                                         \
1539   vec_8x16u colors;                                                            \
1540                                                                                \
1541   u32 color_r = color & 0xFF;                                                  \
1542   u32 color_g = (color >> 8) & 0xFF;                                           \
1543   u32 color_b = (color >> 16) & 0xFF;                                          \
1544                                                                                \
1545   color = (color_r >> 3) | ((color_g >> 3) << 5) |                             \
1546    ((color_b >> 3) << 10);                                                     \
1547   dup_8x16b(colors, color);                                                    \
1548   setup_blocks_variables_unshaded_untextured_##target()                        \
1549
1550 #define setup_blocks_span_initialize_dithered_textured()                       \
1551   vec_8x16u dither_offsets;                                                    \
1552   shl_long_8x8b(dither_offsets, dither_offsets_short, 4)                       \
1553
1554 #define setup_blocks_span_initialize_dithered_untextured()                     \
1555   vec_8x8u dither_offsets;                                                     \
1556   add_8x8b(dither_offsets, dither_offsets_short, d64_4)                        \
1557
1558 #define setup_blocks_span_initialize_dithered(texturing)                       \
1559   u32 dither_row = psx_gpu->dither_table[y & 0x3];                             \
1560   u32 dither_shift = (span_edge_data->left_x & 0x3) * 8;                       \
1561   vec_8x8s dither_offsets_short;                                               \
1562                                                                                \
1563   dither_row =                                                                 \
1564    (dither_row >> dither_shift) | (dither_row << (32 - dither_shift));         \
1565   dup_2x32b(vector_cast(vec_2x32u, dither_offsets_short), dither_row);         \
1566   setup_blocks_span_initialize_dithered_##texturing()                          \
1567
1568 #define setup_blocks_span_initialize_undithered(texturing)                     \
1569
1570
1571 #define setup_blocks_span_initialize_shaded_textured()                         \
1572 {                                                                              \
1573   vec_4x32u block_span;                                                        \
1574   u32 offset = span_edge_data->left_x;                                         \
1575                                                                                \
1576   uvrg = *span_uvrg_offset;                                                    \
1577   mla_scalar_4x32b(uvrg, uvrg_dx, offset);                                     \
1578   b = *span_b_offset;                                                          \
1579   b += b_dx * offset;                                                          \
1580                                                                                \
1581   dup_4x32b(u_block, uvrg.e[0]);                                               \
1582   dup_4x32b(v_block, uvrg.e[1]);                                               \
1583   dup_4x32b(r_block, uvrg.e[2]);                                               \
1584   dup_4x32b(g_block, uvrg.e[3]);                                               \
1585   dup_4x32b(b_block, b);                                                       \
1586                                                                                \
1587   block_span = psx_gpu->u_block_span;                                          \
1588   add_4x32b(u_block, u_block, block_span);                                     \
1589   block_span = psx_gpu->v_block_span;                                          \
1590   add_4x32b(v_block, v_block, block_span);                                     \
1591   block_span = psx_gpu->r_block_span;                                          \
1592   add_4x32b(r_block, r_block, block_span);                                     \
1593   block_span = psx_gpu->g_block_span;                                          \
1594   add_4x32b(g_block, g_block, block_span);                                     \
1595   block_span = psx_gpu->b_block_span;                                          \
1596   add_4x32b(b_block, b_block, block_span);                                     \
1597 }
1598
1599 #define setup_blocks_span_initialize_shaded_untextured()                       \
1600 {                                                                              \
1601   vec_4x32u block_span;                                                        \
1602   u32 offset = span_edge_data->left_x;                                         \
1603                                                                                \
1604   rgb.low = span_uvrg_offset->high;                                            \
1605   rgb.high.e[0] = *span_b_offset;                                              \
1606   mla_scalar_4x32b(rgb, rgb_dx, offset);                                       \
1607                                                                                \
1608   dup_4x32b(r_block, rgb.e[0]);                                                \
1609   dup_4x32b(g_block, rgb.e[1]);                                                \
1610   dup_4x32b(b_block, rgb.e[2]);                                                \
1611                                                                                \
1612   block_span = psx_gpu->r_block_span;                                          \
1613   add_4x32b(r_block, r_block, block_span);                                     \
1614   block_span = psx_gpu->g_block_span;                                          \
1615   add_4x32b(g_block, g_block, block_span);                                     \
1616   block_span = psx_gpu->b_block_span;                                          \
1617   add_4x32b(b_block, b_block, block_span);                                     \
1618 }                                                                              \
1619
1620 #define setup_blocks_span_initialize_unshaded_textured()                       \
1621 {                                                                              \
1622   vec_4x32u block_span;                                                        \
1623   u32 offset = span_edge_data->left_x;                                         \
1624                                                                                \
1625   uv = span_uvrg_offset->low;                                                  \
1626   mla_scalar_2x32b(uv, uv_dx, offset);                                         \
1627                                                                                \
1628   dup_4x32b(u_block, uv.e[0]);                                                 \
1629   dup_4x32b(v_block, uv.e[1]);                                                 \
1630                                                                                \
1631   block_span = psx_gpu->u_block_span;                                          \
1632   add_4x32b(u_block, u_block, block_span);                                     \
1633   block_span = psx_gpu->v_block_span;                                          \
1634   add_4x32b(v_block, v_block, block_span);                                     \
1635 }                                                                              \
1636
1637 #define setup_blocks_span_initialize_unshaded_untextured()                     \
1638
1639
1640 #define setup_blocks_texture_swizzled()                                        \
1641 {                                                                              \
1642   vec_8x8u u_saved = u;                                                        \
1643   sli_8x8b(u, v, 4);                                                           \
1644   sri_8x8b(v, u_saved, 4);                                                     \
1645 }                                                                              \
1646
1647 #define setup_blocks_texture_unswizzled()                                      \
1648
1649 #define setup_blocks_store_shaded_textured(swizzling, dithering, target,       \
1650  edge_type)                                                                    \
1651 {                                                                              \
1652   vec_8x16u u_whole;                                                           \
1653   vec_8x16u v_whole;                                                           \
1654   vec_8x16u r_whole;                                                           \
1655   vec_8x16u g_whole;                                                           \
1656   vec_8x16u b_whole;                                                           \
1657                                                                                \
1658   vec_8x8u u;                                                                  \
1659   vec_8x8u v;                                                                  \
1660   vec_8x8u r;                                                                  \
1661   vec_8x8u g;                                                                  \
1662   vec_8x8u b;                                                                  \
1663   vec_8x16u uv;                                                                \
1664                                                                                \
1665   vec_4x32u dx4;                                                               \
1666   vec_4x32u dx8;                                                               \
1667                                                                                \
1668   shr_narrow_4x32b(u_whole.low, u_block, 16);                                  \
1669   shr_narrow_4x32b(v_whole.low, v_block, 16);                                  \
1670   shr_narrow_4x32b(r_whole.low, r_block, 16);                                  \
1671   shr_narrow_4x32b(g_whole.low, g_block, 16);                                  \
1672   shr_narrow_4x32b(b_whole.low, b_block, 16);                                  \
1673                                                                                \
1674   dup_4x32b(dx4, uvrg_dx4.e[0]);                                               \
1675   add_high_narrow_4x32b(u_whole.high, u_block, dx4);                           \
1676   dup_4x32b(dx4, uvrg_dx4.e[1]);                                               \
1677   add_high_narrow_4x32b(v_whole.high, v_block, dx4);                           \
1678   dup_4x32b(dx4, uvrg_dx4.e[2]);                                               \
1679   add_high_narrow_4x32b(r_whole.high, r_block, dx4);                           \
1680   dup_4x32b(dx4, uvrg_dx4.e[3]);                                               \
1681   add_high_narrow_4x32b(g_whole.high, g_block, dx4);                           \
1682   dup_4x32b(dx4, b_dx4);                                                       \
1683   add_high_narrow_4x32b(b_whole.high, b_block, dx4);                           \
1684                                                                                \
1685   mov_narrow_8x16b(u, u_whole);                                                \
1686   mov_narrow_8x16b(v, v_whole);                                                \
1687   mov_narrow_8x16b(r, r_whole);                                                \
1688   mov_narrow_8x16b(g, g_whole);                                                \
1689   mov_narrow_8x16b(b, b_whole);                                                \
1690                                                                                \
1691   dup_4x32b(dx8, uvrg_dx8.e[0]);                                               \
1692   add_4x32b(u_block, u_block, dx8);                                            \
1693   dup_4x32b(dx8, uvrg_dx8.e[1]);                                               \
1694   add_4x32b(v_block, v_block, dx8);                                            \
1695   dup_4x32b(dx8, uvrg_dx8.e[2]);                                               \
1696   add_4x32b(r_block, r_block, dx8);                                            \
1697   dup_4x32b(dx8, uvrg_dx8.e[3]);                                               \
1698   add_4x32b(g_block, g_block, dx8);                                            \
1699   dup_4x32b(dx8, b_dx8);                                                       \
1700   add_4x32b(b_block, b_block, dx8);                                            \
1701                                                                                \
1702   and_8x8b(u, u, texture_mask.low);                                            \
1703   and_8x8b(v, v, texture_mask.high);                                           \
1704   setup_blocks_texture_##swizzling();                                          \
1705                                                                                \
1706   zip_8x16b(uv, u, v);                                                         \
1707   block->uv = uv;                                                              \
1708   block->r = r;                                                                \
1709   block->g = g;                                                                \
1710   block->b = b;                                                                \
1711   block->dither_offsets = vector_cast(vec_8x16u, dither_offsets);              \
1712   block->fb_ptr = fb_ptr;                                                      \
1713 }                                                                              \
1714
1715 #define setup_blocks_store_unshaded_textured(swizzling, dithering, target,     \
1716  edge_type)                                                                    \
1717 {                                                                              \
1718   vec_8x16u u_whole;                                                           \
1719   vec_8x16u v_whole;                                                           \
1720                                                                                \
1721   vec_8x8u u;                                                                  \
1722   vec_8x8u v;                                                                  \
1723   vec_8x16u uv;                                                                \
1724                                                                                \
1725   vec_4x32u dx4;                                                               \
1726   vec_4x32u dx8;                                                               \
1727                                                                                \
1728   shr_narrow_4x32b(u_whole.low, u_block, 16);                                  \
1729   shr_narrow_4x32b(v_whole.low, v_block, 16);                                  \
1730                                                                                \
1731   dup_4x32b(dx4, uv_dx4.e[0]);                                                 \
1732   add_high_narrow_4x32b(u_whole.high, u_block, dx4);                           \
1733   dup_4x32b(dx4, uv_dx4.e[1]);                                                 \
1734   add_high_narrow_4x32b(v_whole.high, v_block, dx4);                           \
1735                                                                                \
1736   mov_narrow_8x16b(u, u_whole);                                                \
1737   mov_narrow_8x16b(v, v_whole);                                                \
1738                                                                                \
1739   dup_4x32b(dx8, uv_dx8.e[0]);                                                 \
1740   add_4x32b(u_block, u_block, dx8);                                            \
1741   dup_4x32b(dx8, uv_dx8.e[1]);                                                 \
1742   add_4x32b(v_block, v_block, dx8);                                            \
1743                                                                                \
1744   and_8x8b(u, u, texture_mask.low);                                            \
1745   and_8x8b(v, v, texture_mask.high);                                           \
1746   setup_blocks_texture_##swizzling();                                          \
1747                                                                                \
1748   zip_8x16b(uv, u, v);                                                         \
1749   block->uv = uv;                                                              \
1750   block->dither_offsets = vector_cast(vec_8x16u, dither_offsets);              \
1751   block->fb_ptr = fb_ptr;                                                      \
1752 }                                                                              \
1753
1754 #define setup_blocks_store_shaded_untextured_dithered()                        \
1755   addq_8x8b(r, r, dither_offsets);                                             \
1756   addq_8x8b(g, g, dither_offsets);                                             \
1757   addq_8x8b(b, b, dither_offsets);                                             \
1758                                                                                \
1759   subq_8x8b(r, r, d64_4);                                                      \
1760   subq_8x8b(g, g, d64_4);                                                      \
1761   subq_8x8b(b, b, d64_4)                                                       \
1762
1763 #define setup_blocks_store_shaded_untextured_undithered()                      \
1764
1765
1766 #define setup_blocks_store_untextured_pixels_indirect_full(_pixels)            \
1767   block->pixels = _pixels;                                                     \
1768   block->fb_ptr = fb_ptr                                                       \
1769
1770 #define setup_blocks_store_untextured_pixels_indirect_edge(_pixels)            \
1771   block->pixels = _pixels;                                                     \
1772   block->fb_ptr = fb_ptr                                                       \
1773
1774 #define setup_blocks_store_shaded_untextured_seed_pixels_indirect()            \
1775   mul_long_8x8b(pixels, r, d64_1)                                              \
1776
1777
1778 #define setup_blocks_store_untextured_pixels_direct_full(_pixels)              \
1779   store_8x16b(_pixels, fb_ptr)                                                 \
1780
1781 #define setup_blocks_store_untextured_pixels_direct_edge(_pixels)              \
1782 {                                                                              \
1783   vec_8x16u fb_pixels;                                                         \
1784   vec_8x16u draw_mask;                                                         \
1785   vec_8x16u test_mask = psx_gpu->test_mask;                                    \
1786                                                                                \
1787   load_8x16b(fb_pixels, fb_ptr);                                               \
1788   dup_8x16b(draw_mask, span_edge_data->right_mask);                            \
1789   tst_8x16b(draw_mask, draw_mask, test_mask);                                  \
1790   bif_8x16b(fb_pixels, _pixels, draw_mask);                                    \
1791   store_8x16b(fb_pixels, fb_ptr);                                              \
1792 }                                                                              \
1793
1794 #define setup_blocks_store_shaded_untextured_seed_pixels_direct()              \
1795   pixels = msb_mask;                                                           \
1796   mla_long_8x8b(pixels, r, d64_1)                                              \
1797
1798
1799 #define setup_blocks_store_shaded_untextured(swizzling, dithering, target,     \
1800  edge_type)                                                                    \
1801 {                                                                              \
1802   vec_8x16u r_whole;                                                           \
1803   vec_8x16u g_whole;                                                           \
1804   vec_8x16u b_whole;                                                           \
1805                                                                                \
1806   vec_8x8u r;                                                                  \
1807   vec_8x8u g;                                                                  \
1808   vec_8x8u b;                                                                  \
1809                                                                                \
1810   vec_4x32u dx4;                                                               \
1811   vec_4x32u dx8;                                                               \
1812                                                                                \
1813   vec_8x16u pixels;                                                            \
1814                                                                                \
1815   shr_narrow_4x32b(r_whole.low, r_block, 16);                                  \
1816   shr_narrow_4x32b(g_whole.low, g_block, 16);                                  \
1817   shr_narrow_4x32b(b_whole.low, b_block, 16);                                  \
1818                                                                                \
1819   dup_4x32b(dx4, rgb_dx4.e[0]);                                                \
1820   add_high_narrow_4x32b(r_whole.high, r_block, dx4);                           \
1821   dup_4x32b(dx4, rgb_dx4.e[1]);                                                \
1822   add_high_narrow_4x32b(g_whole.high, g_block, dx4);                           \
1823   dup_4x32b(dx4, rgb_dx4.e[2]);                                                \
1824   add_high_narrow_4x32b(b_whole.high, b_block, dx4);                           \
1825                                                                                \
1826   mov_narrow_8x16b(r, r_whole);                                                \
1827   mov_narrow_8x16b(g, g_whole);                                                \
1828   mov_narrow_8x16b(b, b_whole);                                                \
1829                                                                                \
1830   dup_4x32b(dx8, rgb_dx8.e[0]);                                                \
1831   add_4x32b(r_block, r_block, dx8);                                            \
1832   dup_4x32b(dx8, rgb_dx8.e[1]);                                                \
1833   add_4x32b(g_block, g_block, dx8);                                            \
1834   dup_4x32b(dx8, rgb_dx8.e[2]);                                                \
1835   add_4x32b(b_block, b_block, dx8);                                            \
1836                                                                                \
1837   setup_blocks_store_shaded_untextured_##dithering();                          \
1838                                                                                \
1839   shr_8x8b(r, r, 3);                                                           \
1840   bic_8x8b(g, g, d64_0x07);                                                    \
1841   bic_8x8b(b, b, d64_0x07);                                                    \
1842                                                                                \
1843   setup_blocks_store_shaded_untextured_seed_pixels_##target();                 \
1844   mla_long_8x8b(pixels, g, d64_4);                                             \
1845   mla_long_8x8b(pixels, b, d64_128)                                            \
1846                                                                                \
1847   setup_blocks_store_untextured_pixels_##target##_##edge_type(pixels);         \
1848 }                                                                              \
1849
1850 #define setup_blocks_store_unshaded_untextured(swizzling, dithering, target,   \
1851  edge_type)                                                                    \
1852   setup_blocks_store_untextured_pixels_##target##_##edge_type(colors)          \
1853
1854
1855 #define setup_blocks_store_draw_mask_textured_indirect(_block, bits)           \
1856   (_block)->draw_mask_bits = bits                                              \
1857
1858 #define setup_blocks_store_draw_mask_untextured_indirect(_block, bits)         \
1859 {                                                                              \
1860   vec_8x16u bits_mask;                                                         \
1861   vec_8x16u test_mask = psx_gpu->test_mask;                                    \
1862   dup_8x16b(bits_mask, bits);                                                  \
1863   tst_8x16b(bits_mask, bits_mask, test_mask);                                  \
1864   (_block)->draw_mask = bits_mask;                                             \
1865 }                                                                              \
1866
1867 #define setup_blocks_store_draw_mask_untextured_direct(_block, bits)           \
1868
1869
1870 #define setup_blocks_add_blocks_indirect()                                     \
1871   num_blocks += span_num_blocks;                                               \
1872                                                                                \
1873   if(num_blocks > MAX_BLOCKS)                                                  \
1874   {                                                                            \
1875     psx_gpu->num_blocks = num_blocks - span_num_blocks;                        \
1876     flush_render_block_buffer(psx_gpu);                                        \
1877     num_blocks = span_num_blocks;                                              \
1878     block = psx_gpu->blocks;                                                   \
1879   }                                                                            \
1880
1881 #define setup_blocks_add_blocks_direct()                                       \
1882   stats_add(texel_blocks_untextured, span_num_blocks);                         \
1883   span_pixel_blocks += span_num_blocks                                         \
1884
1885
1886 #define setup_blocks_builder(shading, texturing, dithering, sw, target)        \
1887 void setup_blocks_##shading##_##texturing##_##dithering##_##sw##_##target(     \
1888  psx_gpu_struct *psx_gpu)                                                      \
1889 {                                                                              \
1890   setup_blocks_load_msb_mask_##target();                                       \
1891   setup_blocks_variables_##shading##_##texturing(target);                      \
1892                                                                                \
1893   edge_data_struct *span_edge_data = psx_gpu->span_edge_data;                  \
1894   vec_4x32u *span_uvrg_offset = psx_gpu->span_uvrg_offset;                     \
1895   u32 *span_b_offset = psx_gpu->span_b_offset;                                 \
1896                                                                                \
1897   block_struct *block = psx_gpu->blocks + psx_gpu->num_blocks;                 \
1898                                                                                \
1899   u32 num_spans = psx_gpu->num_spans;                                          \
1900                                                                                \
1901   u16 *fb_ptr;                                                                 \
1902   u32 y;                                                                       \
1903                                                                                \
1904   u32 num_blocks = psx_gpu->num_blocks;                                        \
1905   u32 span_num_blocks;                                                         \
1906                                                                                \
1907   while(num_spans)                                                             \
1908   {                                                                            \
1909     span_num_blocks = span_edge_data->num_blocks;                              \
1910     if(span_num_blocks)                                                        \
1911     {                                                                          \
1912       y = span_edge_data->y;                                                   \
1913       fb_ptr = psx_gpu->vram_out_ptr + span_edge_data->left_x + (y * 1024);    \
1914                                                                                \
1915       setup_blocks_span_initialize_##shading##_##texturing();                  \
1916       setup_blocks_span_initialize_##dithering(texturing);                     \
1917                                                                                \
1918       setup_blocks_add_blocks_##target();                                      \
1919                                                                                \
1920       s32 pixel_span = span_num_blocks * 8;                                    \
1921       pixel_span -= __builtin_popcount(span_edge_data->right_mask & 0xFF);     \
1922       span_pixels += pixel_span;                                               \
1923                                                                                \
1924       span_num_blocks--;                                                       \
1925       while(span_num_blocks)                                                   \
1926       {                                                                        \
1927         setup_blocks_store_##shading##_##texturing(sw, dithering, target,      \
1928          full);                                                                \
1929         setup_blocks_store_draw_mask_##texturing##_##target(block, 0x00);      \
1930                                                                                \
1931         fb_ptr += 8;                                                           \
1932         block++;                                                               \
1933         span_num_blocks--;                                                     \
1934       }                                                                        \
1935                                                                                \
1936       setup_blocks_store_##shading##_##texturing(sw, dithering, target, edge); \
1937       setup_blocks_store_draw_mask_##texturing##_##target(block,               \
1938        span_edge_data->right_mask);                                            \
1939                                                                                \
1940       block++;                                                                 \
1941     }                                                                          \
1942     else                                                                       \
1943     {                                                                          \
1944       zero_block_spans++;                                                      \
1945     }                                                                          \
1946                                                                                \
1947     num_spans--;                                                               \
1948     span_edge_data++;                                                          \
1949     span_uvrg_offset++;                                                        \
1950     span_b_offset++;                                                           \
1951   }                                                                            \
1952                                                                                \
1953   psx_gpu->num_blocks = num_blocks;                                            \
1954 }                                                                              \
1955
1956
1957 //setup_blocks_builder(unshaded, untextured, undithered, unswizzled, direct);
1958
1959 #ifndef NEON_BUILD
1960
1961 setup_blocks_builder(shaded, textured, dithered, swizzled, indirect);
1962 setup_blocks_builder(shaded, textured, dithered, unswizzled, indirect);
1963
1964 setup_blocks_builder(unshaded, textured, dithered, unswizzled, indirect);
1965 setup_blocks_builder(unshaded, textured, dithered, swizzled, indirect);
1966
1967 setup_blocks_builder(shaded, untextured, undithered, unswizzled, indirect);
1968 setup_blocks_builder(shaded, untextured, dithered, unswizzled, indirect);
1969 setup_blocks_builder(shaded, untextured, undithered, unswizzled, direct);
1970 setup_blocks_builder(shaded, untextured, dithered, unswizzled, direct);
1971
1972 setup_blocks_builder(unshaded, untextured, undithered, unswizzled, indirect);
1973 setup_blocks_builder(unshaded, untextured, undithered, unswizzled, direct);
1974
1975 void texture_blocks_untextured(psx_gpu_struct *psx_gpu)
1976 {
1977   if(psx_gpu->primitive_type != PRIMITIVE_TYPE_SPRITE)
1978     stats_add(texel_blocks_untextured, psx_gpu->num_blocks);
1979 }
1980
1981 void texture_blocks_4bpp(psx_gpu_struct *psx_gpu)
1982 {
1983   block_struct *block = psx_gpu->blocks;
1984   u32 num_blocks = psx_gpu->num_blocks;
1985   stats_add(texel_blocks_4bpp, num_blocks);
1986
1987   vec_8x8u texels_low;
1988   vec_8x8u texels_high;
1989   vec_8x8u texels;
1990   vec_8x16u pixels;
1991
1992   vec_8x16u clut_a;
1993   vec_8x16u clut_b;
1994   vec_16x8u clut_low;
1995   vec_16x8u clut_high;
1996
1997   u8 *texture_ptr_8bpp = psx_gpu->texture_page_ptr;
1998   u16 *clut_ptr = psx_gpu->clut_ptr;
1999
2000   // Can be done with one deinterleaving load on NEON
2001   load_8x16b(clut_a, clut_ptr);
2002   load_8x16b(clut_b, clut_ptr + 8);
2003   unzip_16x8b(clut_low, clut_high, clut_a, clut_b);
2004
2005   if(psx_gpu->current_texture_mask & psx_gpu->dirty_textures_4bpp_mask)
2006     update_texture_4bpp_cache(psx_gpu);
2007
2008   while(num_blocks)
2009   {
2010     texels.e[0] = texture_ptr_8bpp[block->uv.e[0]];
2011     texels.e[1] = texture_ptr_8bpp[block->uv.e[1]];
2012     texels.e[2] = texture_ptr_8bpp[block->uv.e[2]];
2013     texels.e[3] = texture_ptr_8bpp[block->uv.e[3]];
2014     texels.e[4] = texture_ptr_8bpp[block->uv.e[4]];
2015     texels.e[5] = texture_ptr_8bpp[block->uv.e[5]];
2016     texels.e[6] = texture_ptr_8bpp[block->uv.e[6]];
2017     texels.e[7] = texture_ptr_8bpp[block->uv.e[7]];
2018
2019     tbl_16(texels_low, texels, clut_low);
2020     tbl_16(texels_high, texels, clut_high);
2021
2022     // Can be done with an interleaving store on NEON
2023     zip_8x16b(pixels, texels_low, texels_high);
2024
2025     block->texels = pixels;
2026
2027     num_blocks--;
2028     block++;
2029   }
2030 }
2031
2032 void texture_blocks_8bpp(psx_gpu_struct *psx_gpu)
2033 {
2034   block_struct *block = psx_gpu->blocks;
2035   u32 num_blocks = psx_gpu->num_blocks;
2036
2037   stats_add(texel_blocks_8bpp, num_blocks);
2038
2039   if(psx_gpu->current_texture_mask & psx_gpu->dirty_textures_8bpp_mask)
2040     update_texture_8bpp_cache(psx_gpu);
2041
2042   vec_8x16u texels;
2043   u8 *texture_ptr_8bpp = psx_gpu->texture_page_ptr;
2044
2045   u32 texel;
2046   u32 offset;
2047   u32 i;
2048
2049   while(num_blocks)
2050   {
2051     for(i = 0; i < 8; i++)
2052     {
2053       offset = block->uv.e[i];
2054
2055       texel = texture_ptr_8bpp[offset];
2056       texels.e[i] = psx_gpu->clut_ptr[texel];
2057     }
2058
2059     block->texels = texels;
2060
2061     num_blocks--;
2062     block++;
2063   }
2064 }
2065
2066 void texture_blocks_16bpp(psx_gpu_struct *psx_gpu)
2067 {
2068   block_struct *block = psx_gpu->blocks;
2069   u32 num_blocks = psx_gpu->num_blocks;
2070
2071   stats_add(texel_blocks_16bpp, num_blocks);
2072
2073   vec_8x16u texels;
2074
2075   u16 *texture_ptr_16bpp = psx_gpu->texture_page_ptr;
2076   u32 offset;
2077   u32 i;
2078
2079   while(num_blocks)
2080   {
2081     for(i = 0; i < 8; i++)
2082     {
2083       offset = block->uv.e[i];
2084       offset += ((offset & 0xFF00) * 3);
2085
2086       texels.e[i] = texture_ptr_16bpp[offset];
2087     }
2088
2089     block->texels = texels;
2090
2091     num_blocks--;
2092     block++;
2093   }
2094 }
2095
2096 #endif
2097
2098
2099 #define shade_blocks_load_msb_mask_indirect()                                  \
2100
2101 #define shade_blocks_load_msb_mask_direct()                                    \
2102   vec_8x16u msb_mask;                                                          \
2103   dup_8x16b(msb_mask, psx_gpu->mask_msb);                                      \
2104
2105 #define shade_blocks_store_indirect(_draw_mask, _pixels)                       \
2106   block->draw_mask = _draw_mask;                                               \
2107   block->pixels = _pixels                                                      \
2108
2109 #define shade_blocks_store_direct(_draw_mask, _pixels)                         \
2110 {                                                                              \
2111   vec_8x16u fb_pixels;                                                         \
2112   or_8x16b(_pixels, _pixels, msb_mask);                                        \
2113   load_8x16b(fb_pixels, block->fb_ptr);                                        \
2114   bif_8x16b(fb_pixels, _pixels, _draw_mask);                                   \
2115   store_8x16b(fb_pixels, block->fb_ptr);                                       \
2116 }                                                                              \
2117
2118
2119 #define shade_blocks_textured_false_modulated_check_dithered(target)           \
2120   if(psx_gpu->triangle_color == 0x808080)                                      \
2121   {                                                                            \
2122     false_modulated_blocks += num_blocks;                                      \
2123   }                                                                            \
2124
2125 #define shade_blocks_textured_false_modulated_check_undithered(target)         \
2126   if(psx_gpu->triangle_color == 0x808080)                                      \
2127   {                                                                            \
2128                                                                                \
2129     shade_blocks_textured_unmodulated_##target(psx_gpu);                       \
2130     false_modulated_blocks += num_blocks;                                      \
2131     return;                                                                    \
2132   }                                                                            \
2133
2134
2135 #define shade_blocks_textured_modulated_shaded_primitive_load(dithering,       \
2136  target)                                                                       \
2137
2138 #define shade_blocks_textured_modulated_unshaded_primitive_load(dithering,     \
2139  target)                                                                       \
2140 {                                                                              \
2141   u32 color = psx_gpu->triangle_color;                                         \
2142   dup_8x8b(colors_r, color);                                                   \
2143   dup_8x8b(colors_g, color >> 8);                                              \
2144   dup_8x8b(colors_b, color >> 16);                                             \
2145   shade_blocks_textured_false_modulated_check_##dithering(target);             \
2146 }                                                                              \
2147
2148 #define shade_blocks_textured_modulated_shaded_block_load()                    \
2149   colors_r = block->r;                                                         \
2150   colors_g = block->g;                                                         \
2151   colors_b = block->b                                                          \
2152
2153 #define shade_blocks_textured_modulated_unshaded_block_load()                  \
2154
2155 #define shade_blocks_textured_modulate_dithered(component)                     \
2156   pixels_##component = block->dither_offsets;                                  \
2157   mla_long_8x8b(pixels_##component, texels_##component, colors_##component)    \
2158
2159 #define shade_blocks_textured_modulate_undithered(component)                   \
2160   mul_long_8x8b(pixels_##component, texels_##component, colors_##component)    \
2161
2162 #define shade_blocks_textured_modulated_builder(shading, dithering, target)    \
2163 void shade_blocks_##shading##_textured_modulated_##dithering##_##target(       \
2164  psx_gpu_struct *psx_gpu)                                                      \
2165 {                                                                              \
2166   block_struct *block = psx_gpu->blocks;                                       \
2167   u32 num_blocks = psx_gpu->num_blocks;                                        \
2168   vec_8x16u texels;                                                            \
2169                                                                                \
2170   vec_8x8u texels_r;                                                           \
2171   vec_8x8u texels_g;                                                           \
2172   vec_8x8u texels_b;                                                           \
2173                                                                                \
2174   vec_8x8u colors_r;                                                           \
2175   vec_8x8u colors_g;                                                           \
2176   vec_8x8u colors_b;                                                           \
2177                                                                                \
2178   vec_8x8u pixels_r_low;                                                       \
2179   vec_8x8u pixels_g_low;                                                       \
2180   vec_8x8u pixels_b_low;                                                       \
2181   vec_8x16u pixels;                                                            \
2182                                                                                \
2183   vec_8x16u pixels_r;                                                          \
2184   vec_8x16u pixels_g;                                                          \
2185   vec_8x16u pixels_b;                                                          \
2186                                                                                \
2187   vec_8x16u draw_mask;                                                         \
2188   vec_8x16u zero_mask;                                                         \
2189                                                                                \
2190   vec_8x8u d64_0x07;                                                           \
2191   vec_8x8u d64_0x1F;                                                           \
2192   vec_8x8u d64_1;                                                              \
2193   vec_8x8u d64_4;                                                              \
2194   vec_8x8u d64_128;                                                            \
2195                                                                                \
2196   vec_8x16u d128_0x8000;                                                       \
2197                                                                                \
2198   vec_8x16u test_mask = psx_gpu->test_mask;                                    \
2199   u32 draw_mask_bits;                                                          \
2200   shade_blocks_load_msb_mask_##target();                                       \
2201                                                                                \
2202   dup_8x8b(d64_0x07, 0x07);                                                    \
2203   dup_8x8b(d64_0x1F, 0x1F);                                                    \
2204   dup_8x8b(d64_1, 1);                                                          \
2205   dup_8x8b(d64_4, 4);                                                          \
2206   dup_8x8b(d64_128, 128);                                                      \
2207                                                                                \
2208   dup_8x16b(d128_0x8000, 0x8000);                                              \
2209                                                                                \
2210   shade_blocks_textured_modulated_##shading##_primitive_load(dithering,        \
2211    target);                                                                    \
2212                                                                                \
2213   while(num_blocks)                                                            \
2214   {                                                                            \
2215     draw_mask_bits = block->draw_mask_bits;                                    \
2216     dup_8x16b(draw_mask, draw_mask_bits);                                      \
2217     tst_8x16b(draw_mask, draw_mask, test_mask);                                \
2218                                                                                \
2219     shade_blocks_textured_modulated_##shading##_block_load();                  \
2220                                                                                \
2221     texels = block->texels;                                                    \
2222                                                                                \
2223     mov_narrow_8x16b(texels_r, texels);                                        \
2224     shr_narrow_8x16b(texels_g, texels, 5);                                     \
2225     shr_narrow_8x16b(texels_b, texels, 7);                                     \
2226                                                                                \
2227     and_8x8b(texels_r, texels_r, d64_0x1F);                                    \
2228     and_8x8b(texels_g, texels_g, d64_0x1F);                                    \
2229     shr_8x8b(texels_b, texels_b, 3);                                           \
2230                                                                                \
2231     shade_blocks_textured_modulate_##dithering(r);                             \
2232     shade_blocks_textured_modulate_##dithering(g);                             \
2233     shade_blocks_textured_modulate_##dithering(b);                             \
2234                                                                                \
2235     cmpeqz_8x16b(zero_mask, texels);                                           \
2236     and_8x16b(pixels, texels, d128_0x8000);                                    \
2237                                                                                \
2238     shrq_narrow_signed_8x16b(pixels_r_low, pixels_r, 4);                       \
2239     shrq_narrow_signed_8x16b(pixels_g_low, pixels_g, 4);                       \
2240     shrq_narrow_signed_8x16b(pixels_b_low, pixels_b, 4);                       \
2241                                                                                \
2242     or_8x16b(zero_mask, draw_mask, zero_mask);                                 \
2243                                                                                \
2244     shr_8x8b(pixels_r_low, pixels_r_low, 3);                                   \
2245     bic_8x8b(pixels_g_low, pixels_g_low, d64_0x07);                            \
2246     bic_8x8b(pixels_b_low, pixels_b_low, d64_0x07);                            \
2247                                                                                \
2248     mla_long_8x8b(pixels, pixels_r_low, d64_1);                                \
2249     mla_long_8x8b(pixels, pixels_g_low, d64_4);                                \
2250     mla_long_8x8b(pixels, pixels_b_low, d64_128);                              \
2251                                                                                \
2252     shade_blocks_store_##target(zero_mask, pixels);                            \
2253                                                                                \
2254     num_blocks--;                                                              \
2255     block++;                                                                   \
2256   }                                                                            \
2257 }                                                                              \
2258
2259 #ifndef NEON_BUILD
2260
2261 shade_blocks_textured_modulated_builder(shaded, dithered, direct);
2262 shade_blocks_textured_modulated_builder(shaded, undithered, direct);
2263 shade_blocks_textured_modulated_builder(unshaded, dithered, direct);
2264 shade_blocks_textured_modulated_builder(unshaded, undithered, direct);
2265
2266 shade_blocks_textured_modulated_builder(shaded, dithered, indirect);
2267 shade_blocks_textured_modulated_builder(shaded, undithered, indirect);
2268 shade_blocks_textured_modulated_builder(unshaded, dithered, indirect);
2269 shade_blocks_textured_modulated_builder(unshaded, undithered, indirect);
2270
2271 #endif
2272
2273
2274 #define shade_blocks_textured_unmodulated_builder(target)                      \
2275 void shade_blocks_textured_unmodulated_##target(psx_gpu_struct *psx_gpu)       \
2276 {                                                                              \
2277   block_struct *block = psx_gpu->blocks;                                       \
2278   u32 num_blocks = psx_gpu->num_blocks;                                        \
2279   vec_8x16u draw_mask;                                                         \
2280   vec_8x16u test_mask = psx_gpu->test_mask;                                    \
2281   u32 draw_mask_bits;                                                          \
2282                                                                                \
2283   vec_8x16u pixels;                                                            \
2284   shade_blocks_load_msb_mask_##target();                                       \
2285                                                                                \
2286   while(num_blocks)                                                            \
2287   {                                                                            \
2288     vec_8x16u zero_mask;                                                       \
2289                                                                                \
2290     draw_mask_bits = block->draw_mask_bits;                                    \
2291     dup_8x16b(draw_mask, draw_mask_bits);                                      \
2292     tst_8x16b(draw_mask, draw_mask, test_mask);                                \
2293                                                                                \
2294     pixels = block->texels;                                                    \
2295                                                                                \
2296     cmpeqz_8x16b(zero_mask, pixels);                                           \
2297     or_8x16b(zero_mask, draw_mask, zero_mask);                                 \
2298                                                                                \
2299     shade_blocks_store_##target(zero_mask, pixels);                            \
2300                                                                                \
2301     num_blocks--;                                                              \
2302     block++;                                                                   \
2303   }                                                                            \
2304 }                                                                              \
2305
2306 #define shade_blocks_textured_unmodulated_dithered_builder(target)             \
2307 void shade_blocks_textured_unmodulated_dithered_##target(psx_gpu_struct        \
2308  *psx_gpu)                                                                     \
2309 {                                                                              \
2310   block_struct *block = psx_gpu->blocks;                                       \
2311   u32 num_blocks = psx_gpu->num_blocks;                                        \
2312   vec_8x16u draw_mask;                                                         \
2313   vec_8x16u test_mask = psx_gpu->test_mask;                                    \
2314   u32 draw_mask_bits;                                                          \
2315                                                                                \
2316   vec_8x16u pixels;                                                            \
2317   shade_blocks_load_msb_mask_##target();                                       \
2318                                                                                \
2319   while(num_blocks)                                                            \
2320   {                                                                            \
2321     vec_8x16u zero_mask;                                                       \
2322                                                                                \
2323     draw_mask_bits = block->draw_mask_bits;                                    \
2324     dup_8x16b(draw_mask, draw_mask_bits);                                      \
2325     tst_8x16b(draw_mask, draw_mask, test_mask);                                \
2326                                                                                \
2327     pixels = block->texels;                                                    \
2328                                                                                \
2329     cmpeqz_8x16b(zero_mask, pixels);                                           \
2330     or_8x16b(zero_mask, draw_mask, zero_mask);                                 \
2331                                                                                \
2332     shade_blocks_store_##target(zero_mask, pixels);                            \
2333                                                                                \
2334     num_blocks--;                                                              \
2335     block++;                                                                   \
2336   }                                                                            \
2337 }                                                                              \
2338
2339 #ifndef NEON_BUILD
2340
2341 shade_blocks_textured_unmodulated_builder(indirect)
2342 shade_blocks_textured_unmodulated_builder(direct)
2343
2344 void shade_blocks_unshaded_untextured_indirect(psx_gpu_struct *psx_gpu)
2345 {
2346 }
2347
2348 void shade_blocks_unshaded_untextured_direct(psx_gpu_struct *psx_gpu)
2349 {
2350   block_struct *block = psx_gpu->blocks;
2351   u32 num_blocks = psx_gpu->num_blocks;
2352
2353   vec_8x16u pixels = block->pixels;
2354   shade_blocks_load_msb_mask_direct();
2355
2356   while(num_blocks)
2357   {
2358     shade_blocks_store_direct(block->draw_mask, pixels);
2359
2360     num_blocks--;
2361     block++;
2362   }
2363 }
2364
2365 #endif
2366
2367 void shade_blocks_shaded_untextured(psx_gpu_struct *psx_gpu)
2368 {
2369 }
2370
2371
2372 #define blend_blocks_mask_evaluate_on()                                        \
2373   vec_8x16u mask_pixels;                                                       \
2374   cmpltz_8x16b(mask_pixels, framebuffer_pixels);                               \
2375   or_8x16b(draw_mask, draw_mask, mask_pixels)                                  \
2376
2377 #define blend_blocks_mask_evaluate_off()                                       \
2378
2379 #define blend_blocks_average()                                                 \
2380 {                                                                              \
2381   vec_8x16u pixels_no_msb;                                                     \
2382   vec_8x16u fb_pixels_no_msb;                                                  \
2383                                                                                \
2384   vec_8x16u d128_0x0421;                                                       \
2385   vec_8x16u d128_0x8000;                                                       \
2386                                                                                \
2387   dup_8x16b(d128_0x0421, 0x0421);                                              \
2388   dup_8x16b(d128_0x8000, 0x8000);                                              \
2389                                                                                \
2390   eor_8x16b(blend_pixels, pixels, framebuffer_pixels);                         \
2391   bic_8x16b(pixels_no_msb, pixels, d128_0x8000);                               \
2392   and_8x16b(blend_pixels, blend_pixels, d128_0x0421);                          \
2393   sub_8x16b(blend_pixels, pixels_no_msb, blend_pixels);                        \
2394   bic_8x16b(fb_pixels_no_msb, framebuffer_pixels, d128_0x8000);                \
2395   average_8x16b(blend_pixels, fb_pixels_no_msb, blend_pixels);                 \
2396 }                                                                              \
2397
2398 #define blend_blocks_add()                                                     \
2399 {                                                                              \
2400   vec_8x16u pixels_rb, pixels_g;                                               \
2401   vec_8x16u fb_rb, fb_g;                                                       \
2402                                                                                \
2403   vec_8x16u d128_0x7C1F;                                                       \
2404   vec_8x16u d128_0x03E0;                                                       \
2405                                                                                \
2406   dup_8x16b(d128_0x7C1F, 0x7C1F);                                              \
2407   dup_8x16b(d128_0x03E0, 0x03E0);                                              \
2408                                                                                \
2409   and_8x16b(pixels_rb, pixels, d128_0x7C1F);                                   \
2410   and_8x16b(pixels_g, pixels, d128_0x03E0);                                    \
2411                                                                                \
2412   and_8x16b(fb_rb, framebuffer_pixels, d128_0x7C1F);                           \
2413   and_8x16b(fb_g, framebuffer_pixels, d128_0x03E0);                            \
2414                                                                                \
2415   add_8x16b(fb_rb, fb_rb, pixels_rb);                                          \
2416   add_8x16b(fb_g, fb_g, pixels_g);                                             \
2417                                                                                \
2418   min_16x8b(vector_cast(vec_16x8u, fb_rb), vector_cast(vec_16x8u, fb_rb),      \
2419    vector_cast(vec_16x8u, d128_0x7C1F));                                       \
2420   min_8x16b(fb_g, fb_g, d128_0x03E0);                                          \
2421                                                                                \
2422   or_8x16b(blend_pixels, fb_rb, fb_g);                                         \
2423 }                                                                              \
2424
2425 #define blend_blocks_subtract()                                                \
2426 {                                                                              \
2427   vec_8x16u pixels_rb, pixels_g;                                               \
2428   vec_8x16u fb_rb, fb_g;                                                       \
2429                                                                                \
2430   vec_8x16u d128_0x7C1F;                                                       \
2431   vec_8x16u d128_0x03E0;                                                       \
2432                                                                                \
2433   dup_8x16b(d128_0x7C1F, 0x7C1F);                                              \
2434   dup_8x16b(d128_0x03E0, 0x03E0);                                              \
2435                                                                                \
2436   and_8x16b(pixels_rb, pixels, d128_0x7C1F);                                   \
2437   and_8x16b(pixels_g, pixels, d128_0x03E0);                                    \
2438                                                                                \
2439   and_8x16b(fb_rb, framebuffer_pixels, d128_0x7C1F);                           \
2440   and_8x16b(fb_g, framebuffer_pixels, d128_0x03E0);                            \
2441                                                                                \
2442   subs_16x8b(vector_cast(vec_16x8u, fb_rb),                                    \
2443    vector_cast(vec_16x8u, fb_rb), vector_cast(vec_16x8u, pixels_rb));          \
2444   subs_8x16b(fb_g, fb_g, pixels_g);                                            \
2445                                                                                \
2446   or_8x16b(blend_pixels, fb_rb, fb_g);                                         \
2447 }                                                                              \
2448
2449 #define blend_blocks_add_fourth()                                              \
2450 {                                                                              \
2451   vec_8x16u pixels_rb, pixels_g;                                               \
2452   vec_8x16u pixels_fourth;                                                     \
2453   vec_8x16u fb_rb, fb_g;                                                       \
2454                                                                                \
2455   vec_8x16u d128_0x7C1F;                                                       \
2456   vec_8x16u d128_0x1C07;                                                       \
2457   vec_8x16u d128_0x03E0;                                                       \
2458   vec_8x16u d128_0x00E0;                                                       \
2459                                                                                \
2460   dup_8x16b(d128_0x7C1F, 0x7C1F);                                              \
2461   dup_8x16b(d128_0x1C07, 0x1C07);                                              \
2462   dup_8x16b(d128_0x03E0, 0x03E0);                                              \
2463   dup_8x16b(d128_0x00E0, 0x00E0);                                              \
2464                                                                                \
2465   shr_8x16b(pixels_fourth, vector_cast(vec_8x16s, pixels), 2);                 \
2466                                                                                \
2467   and_8x16b(fb_rb, framebuffer_pixels, d128_0x7C1F);                           \
2468   and_8x16b(fb_g, framebuffer_pixels, d128_0x03E0);                            \
2469                                                                                \
2470   and_8x16b(pixels_rb, pixels_fourth, d128_0x1C07);                            \
2471   and_8x16b(pixels_g, pixels_fourth, d128_0x00E0);                             \
2472                                                                                \
2473   add_8x16b(fb_rb, fb_rb, pixels_rb);                                          \
2474   add_8x16b(fb_g, fb_g, pixels_g);                                             \
2475                                                                                \
2476   min_16x8b(vector_cast(vec_16x8u, fb_rb), vector_cast(vec_16x8u, fb_rb),      \
2477    vector_cast(vec_16x8u, d128_0x7C1F));                                       \
2478   min_8x16b(fb_g, fb_g, d128_0x03E0);                                          \
2479                                                                                \
2480   or_8x16b(blend_pixels, fb_rb, fb_g);                                         \
2481 }                                                                              \
2482
2483 #define blend_blocks_blended_combine_textured()                                \
2484 {                                                                              \
2485   vec_8x16u blend_mask;                                                        \
2486   cmpltz_8x16b(blend_mask, pixels);                                            \
2487                                                                                \
2488   or_immediate_8x16b(blend_pixels, blend_pixels, 0x8000);                      \
2489   bif_8x16b(blend_pixels, pixels, blend_mask);                                 \
2490 }                                                                              \
2491
2492 #define blend_blocks_blended_combine_untextured()                              \
2493
2494
2495 #define blend_blocks_body_blend(blend_mode, texturing)                         \
2496 {                                                                              \
2497   blend_blocks_##blend_mode();                                                 \
2498   blend_blocks_blended_combine_##texturing();                                  \
2499 }                                                                              \
2500
2501 #define blend_blocks_body_average(texturing)                                   \
2502   blend_blocks_body_blend(average, texturing)                                  \
2503
2504 #define blend_blocks_body_add(texturing)                                       \
2505   blend_blocks_body_blend(add, texturing)                                      \
2506
2507 #define blend_blocks_body_subtract(texturing)                                  \
2508   blend_blocks_body_blend(subtract, texturing)                                 \
2509
2510 #define blend_blocks_body_add_fourth(texturing)                                \
2511   blend_blocks_body_blend(add_fourth, texturing)                               \
2512
2513 #define blend_blocks_body_unblended(texturing)                                 \
2514   blend_pixels = pixels                                                        \
2515
2516
2517 #define blend_blocks_builder(texturing, blend_mode, mask_evaluate)             \
2518 void                                                                           \
2519  blend_blocks_##texturing##_##blend_mode##_##mask_evaluate(psx_gpu_struct      \
2520  *psx_gpu)                                                                     \
2521 {                                                                              \
2522   block_struct *block = psx_gpu->blocks;                                       \
2523   u32 num_blocks = psx_gpu->num_blocks;                                        \
2524   vec_8x16u draw_mask;                                                         \
2525   vec_8x16u pixels;                                                            \
2526   vec_8x16u blend_pixels;                                                      \
2527   vec_8x16u framebuffer_pixels;                                                \
2528   vec_8x16u msb_mask;                                                          \
2529                                                                                \
2530   u16 *fb_ptr;                                                                 \
2531                                                                                \
2532   dup_8x16b(msb_mask, psx_gpu->mask_msb);                                      \
2533                                                                                \
2534   while(num_blocks)                                                            \
2535   {                                                                            \
2536     pixels = block->pixels;                                                    \
2537     draw_mask = block->draw_mask;                                              \
2538     fb_ptr = block->fb_ptr;                                                    \
2539                                                                                \
2540     load_8x16b(framebuffer_pixels, fb_ptr);                                    \
2541                                                                                \
2542     blend_blocks_mask_evaluate_##mask_evaluate();                              \
2543     blend_blocks_body_##blend_mode(texturing);                                 \
2544                                                                                \
2545     or_8x16b(blend_pixels, blend_pixels, msb_mask);                            \
2546     bif_8x16b(framebuffer_pixels, blend_pixels, draw_mask);                    \
2547     store_8x16b(framebuffer_pixels, fb_ptr);                                   \
2548                                                                                \
2549     blend_blocks++;                                                            \
2550     num_blocks--;                                                              \
2551     block++;                                                                   \
2552   }                                                                            \
2553 }                                                                              \
2554
2555 #ifndef NEON_BUILD
2556
2557 void blend_blocks_textured_unblended_off(psx_gpu_struct *psx_gpu)
2558 {
2559 }
2560
2561 blend_blocks_builder(textured, average, off);
2562 blend_blocks_builder(textured, average, on);
2563 blend_blocks_builder(textured, add, off);
2564 blend_blocks_builder(textured, add, on);
2565 blend_blocks_builder(textured, subtract, off);
2566 blend_blocks_builder(textured, subtract, on);
2567 blend_blocks_builder(textured, add_fourth, off);
2568 blend_blocks_builder(textured, add_fourth, on);
2569
2570 blend_blocks_builder(untextured, average, off);
2571 blend_blocks_builder(untextured, average, on);
2572 blend_blocks_builder(untextured, add, off);
2573 blend_blocks_builder(untextured, add, on);
2574 blend_blocks_builder(untextured, subtract, off);
2575 blend_blocks_builder(untextured, subtract, on);
2576 blend_blocks_builder(untextured, add_fourth, off);
2577 blend_blocks_builder(untextured, add_fourth, on);
2578
2579 blend_blocks_builder(textured, unblended, on);
2580
2581 #endif
2582
2583
2584 #define vertex_swap(_a, _b)                                                    \
2585 {                                                                              \
2586   vertex_struct *temp_vertex = _a;                                             \
2587   _a = _b;                                                                     \
2588   _b = temp_vertex;                                                            \
2589   triangle_winding ^= 1;                                                       \
2590 }                                                                              \
2591
2592
2593 // Setup blocks parametric-variables:
2594 // SHADE  TEXTURE_MAP SWIZZLING
2595 // 0      0           x
2596 // 0      1           0
2597 // 0      1           1
2598 // 1      0           x
2599 // 1      1           0
2600 // 1      1           1
2601 // 8 inputs, 6 combinations
2602
2603 #define setup_blocks_switch_untextured_unshaded(dithering, target)             \
2604   setup_blocks_unshaded_untextured_undithered_unswizzled_##target              \
2605
2606 #define setup_blocks_switch_untextured_shaded(dithering, target)               \
2607   setup_blocks_shaded_untextured_##dithering##_unswizzled_##target             \
2608
2609 #define setup_blocks_switch_untextured(shading, texture_mode, dithering,       \
2610  target)                                                                       \
2611   setup_blocks_switch_untextured_##shading(dithering, target)                  \
2612
2613 #define setup_blocks_switch_texture_mode_4bpp(shading)                         \
2614   setup_blocks_##shading##_textured_dithered_swizzled_indirect                 \
2615
2616 #define setup_blocks_switch_texture_mode_8bpp(shading)                         \
2617   setup_blocks_##shading##_textured_dithered_swizzled_indirect                 \
2618
2619 #define setup_blocks_switch_texture_mode_16bpp(shading)                        \
2620   setup_blocks_##shading##_textured_dithered_unswizzled_indirect               \
2621
2622 #define setup_blocks_switch_textured(shading, texture_mode, dithering, target) \
2623   setup_blocks_switch_texture_mode_##texture_mode(shading)                     \
2624
2625 #define setup_blocks_switch_blended(shading, texturing, texture_mode,          \
2626  dithering, mask_evaluate)                                                     \
2627   setup_blocks_switch_##texturing(shading, texture_mode, dithering, indirect)  \
2628
2629 #define setup_blocks_switch_unblended_on(shading, texturing, texture_mode,     \
2630  dithering)                                                                    \
2631   setup_blocks_switch_##texturing(shading, texture_mode, dithering, indirect)  \
2632
2633 #define setup_blocks_switch_unblended_off(shading, texturing, texture_mode,    \
2634  dithering)                                                                    \
2635   setup_blocks_switch_##texturing(shading, texture_mode, dithering, direct)    \
2636
2637 #define setup_blocks_switch_unblended(shading, texturing, texture_mode,        \
2638  dithering, mask_evaluate)                                                     \
2639   setup_blocks_switch_unblended_##mask_evaluate(shading, texturing,            \
2640    texture_mode, dithering)                                                    \
2641
2642 #define setup_blocks_switch(shading, texturing, texture_mode, dithering,       \
2643  blending, mask_evaluate)                                                      \
2644   setup_blocks_switch_##blending(shading, texturing, texture_mode,             \
2645    dithering, mask_evaluate)                                                   \
2646
2647
2648 // Texture blocks:
2649
2650 #define texture_blocks_switch_untextured(texture_mode)                         \
2651   texture_blocks_untextured                                                    \
2652
2653 #define texture_blocks_switch_textured(texture_mode)                           \
2654   texture_blocks_##texture_mode                                                \
2655
2656 #define texture_blocks_switch(texturing, texture_mode)                         \
2657   texture_blocks_switch_##texturing(texture_mode)                              \
2658
2659
2660 // Shade blocks parametric-variables:
2661 // SHADE  TEXTURE_MAP  MODULATE_TEXELS  dither_mode
2662 // 0      0            x                x
2663 // 0      1            0                0
2664 // 0      1            0                1
2665 // x      1            1                x
2666 // 1      0            x                0
2667 // 1      0            x                1
2668 // 1      1            0                0
2669 // 1      1            0                1
2670 // 16 inputs, 8 combinations
2671
2672 #define shade_blocks_switch_unshaded_untextured(modulation, dithering, target) \
2673   shade_blocks_unshaded_untextured_##target                                    \
2674
2675 #define shade_blocks_switch_unshaded_textured_unmodulated(dithering, target)   \
2676   shade_blocks_textured_unmodulated_##target                                   \
2677
2678 #define shade_blocks_switch_unshaded_textured_modulated(dithering, target)     \
2679   shade_blocks_unshaded_textured_modulated_##dithering##_##target              \
2680
2681 #define shade_blocks_switch_unshaded_textured(modulation, dithering, target)   \
2682   shade_blocks_switch_unshaded_textured_##modulation(dithering, target)        \
2683
2684 #define shade_blocks_switch_unshaded(texturing, modulation, dithering, target) \
2685   shade_blocks_switch_unshaded_##texturing(modulation, dithering, target)      \
2686
2687 #define shade_blocks_switch_shaded_untextured(modulation, dithering, target)   \
2688   shade_blocks_shaded_untextured                                               \
2689
2690 #define shade_blocks_switch_shaded_textured_unmodulated(dithering, target)     \
2691   shade_blocks_textured_unmodulated_##target                                   \
2692
2693 #define shade_blocks_switch_shaded_textured_modulated(dithering, target)       \
2694   shade_blocks_shaded_textured_modulated_##dithering##_##target                \
2695
2696 #define shade_blocks_switch_shaded_textured(modulation, dithering, target)     \
2697   shade_blocks_switch_shaded_textured_##modulation(dithering, target)          \
2698
2699 #define shade_blocks_switch_shaded(texturing, modulation, dithering, target)   \
2700   shade_blocks_switch_shaded_##texturing(modulation, dithering, target)        \
2701
2702 #define shade_blocks_switch_mask_off(shading, texturing, modulation,           \
2703  dithering)                                                                    \
2704   shade_blocks_switch_##shading(texturing, modulation, dithering, direct)      \
2705
2706 #define shade_blocks_switch_mask_on(shading, texturing, modulation,            \
2707  dithering)                                                                    \
2708   shade_blocks_switch_##shading(texturing, modulation, dithering, indirect)    \
2709
2710 #define shade_blocks_switch_blended(shading, texturing, modulation, dithering, \
2711  mask_evaluate)                                                                \
2712   shade_blocks_switch_##shading(texturing, modulation, dithering, indirect)    \
2713
2714 #define shade_blocks_switch_unblended(shading, texturing, modulation,          \
2715  dithering, mask_evaluate)                                                     \
2716   shade_blocks_switch_mask_##mask_evaluate(shading, texturing, modulation,     \
2717    dithering)                                                                  \
2718
2719 #define shade_blocks_switch(shading, texturing, modulation, dithering,         \
2720  blending, mask_evaluate)                                                      \
2721   shade_blocks_switch_##blending(shading, texturing, modulation, dithering,    \
2722    mask_evaluate)                                                              \
2723
2724
2725 // Blend blocks parametric-variables:
2726 // TEXTURE_MAP BLEND  BM_A BM_B mask_evaluate
2727 // x           0      x    x    0
2728 // x           0      x    x    1
2729 // 0           1      0    0    0
2730 // 0           1      0    0    1
2731 // 0           1      0    1    0
2732 // 0           1      0    1    1
2733 // 0           1      1    0    0
2734 // 0           1      1    0    1
2735 // 0           1      1    1    0
2736 // 0           1      1    1    1
2737 // 1           1      0    0    0
2738 // 1           1      0    0    1
2739 // 1           1      0    1    0
2740 // 1           1      0    1    1
2741 // 1           1      1    0    0
2742 // 1           1      1    0    1
2743 // 1           1      1    1    0
2744 // 1           1      1    1    1
2745 // 32 inputs, 18 combinations
2746
2747 #define blend_blocks_switch_unblended(texturing, blend_mode, mask_evaluate)    \
2748   blend_blocks_textured_unblended_##mask_evaluate                              \
2749
2750 #define blend_blocks_switch_blended(texturing, blend_mode, mask_evaluate)      \
2751   blend_blocks_##texturing##_##blend_mode##_##mask_evaluate                    \
2752
2753 #define blend_blocks_switch(texturing, blending, blend_mode, mask_evaluate)    \
2754   blend_blocks_switch_##blending(texturing, blend_mode, mask_evaluate)         \
2755
2756
2757 #define render_blocks_switch_block_modulation(texture_mode, blend_mode,        \
2758  mask_evaluate, shading, dithering, texturing, blending, modulation)           \
2759 {                                                                              \
2760   setup_blocks_switch(shading, texturing, texture_mode, dithering, blending,   \
2761    mask_evaluate),                                                             \
2762   texture_blocks_switch(texturing, texture_mode),                              \
2763   shade_blocks_switch(shading, texturing, modulation, dithering, blending,     \
2764    mask_evaluate),                                                             \
2765   blend_blocks_switch(texturing, blending, blend_mode, mask_evaluate)          \
2766 }                                                                              \
2767
2768 #define render_blocks_switch_block_blending(texture_mode, blend_mode,          \
2769  mask_evaluate, shading, dithering, texturing, blending)                       \
2770   render_blocks_switch_block_modulation(texture_mode, blend_mode,              \
2771    mask_evaluate, shading, dithering, texturing, blending, modulated),         \
2772   render_blocks_switch_block_modulation(texture_mode, blend_mode,              \
2773    mask_evaluate, shading, dithering, texturing, blending, unmodulated)        \
2774
2775 #define render_blocks_switch_block_texturing(texture_mode, blend_mode,         \
2776  mask_evaluate, shading, dithering, texturing)                                 \
2777   render_blocks_switch_block_blending(texture_mode, blend_mode,                \
2778    mask_evaluate, shading, dithering, texturing, unblended),                   \
2779   render_blocks_switch_block_blending(texture_mode, blend_mode,                \
2780    mask_evaluate, shading, dithering, texturing, blended)                      \
2781
2782 #define render_blocks_switch_block_dithering(texture_mode, blend_mode,         \
2783  mask_evaluate, shading, dithering)                                            \
2784   render_blocks_switch_block_texturing(texture_mode, blend_mode,               \
2785    mask_evaluate, shading, dithering, untextured),                             \
2786   render_blocks_switch_block_texturing(texture_mode, blend_mode,               \
2787    mask_evaluate, shading, dithering, textured)                                \
2788
2789 #define render_blocks_switch_block_shading(texture_mode, blend_mode,           \
2790  mask_evaluate, shading)                                                       \
2791   render_blocks_switch_block_dithering(texture_mode, blend_mode,               \
2792    mask_evaluate, shading, undithered),                                        \
2793   render_blocks_switch_block_dithering(texture_mode, blend_mode,               \
2794    mask_evaluate, shading, dithered)                                           \
2795
2796 #define render_blocks_switch_block_mask_evaluate(texture_mode, blend_mode,     \
2797  mask_evaluate)                                                                \
2798   render_blocks_switch_block_shading(texture_mode, blend_mode, mask_evaluate,  \
2799    unshaded),                                                                  \
2800   render_blocks_switch_block_shading(texture_mode, blend_mode, mask_evaluate,  \
2801    shaded)                                                                     \
2802
2803 #define render_blocks_switch_block_blend_mode(texture_mode, blend_mode)        \
2804   render_blocks_switch_block_mask_evaluate(texture_mode, blend_mode, off),     \
2805   render_blocks_switch_block_mask_evaluate(texture_mode, blend_mode, on)       \
2806
2807 #define render_blocks_switch_block_texture_mode(texture_mode)                  \
2808   render_blocks_switch_block_blend_mode(texture_mode, average),                \
2809   render_blocks_switch_block_blend_mode(texture_mode, add),                    \
2810   render_blocks_switch_block_blend_mode(texture_mode, subtract),               \
2811   render_blocks_switch_block_blend_mode(texture_mode, add_fourth)              \
2812
2813 #define render_blocks_switch_block()                                           \
2814   render_blocks_switch_block_texture_mode(4bpp),                               \
2815   render_blocks_switch_block_texture_mode(8bpp),                               \
2816   render_blocks_switch_block_texture_mode(16bpp),                              \
2817   render_blocks_switch_block_texture_mode(16bpp)                               \
2818
2819
2820 render_block_handler_struct render_triangle_block_handlers[] =
2821 {
2822   render_blocks_switch_block()
2823 };
2824
2825 #undef render_blocks_switch_block_modulation
2826
2827 #define render_blocks_switch_block_modulation(texture_mode, blend_mode,        \
2828  mask_evaluate, shading, dithering, texturing, blending, modulation)           \
2829   "render flags:\n"                                                            \
2830   "texture mode:     " #texture_mode "\n"                                      \
2831   "blend mode:       " #blend_mode "\n"                                        \
2832   "mask evaluation:  " #mask_evaluate "\n"                                     \
2833   #shading "\n"                                                                \
2834   #dithering "\n"                                                              \
2835   #texturing "\n"                                                              \
2836   #blending "\n"                                                               \
2837   #modulation "\n"                                                             \
2838
2839 char *render_block_flag_strings[] =
2840 {
2841   render_blocks_switch_block()
2842 };
2843
2844
2845 #define triangle_y_direction_up   1
2846 #define triangle_y_direction_flat 2
2847 #define triangle_y_direction_down 0
2848
2849 #define triangle_winding_positive 0
2850 #define triangle_winding_negative 1
2851
2852 #define triangle_set_direction(direction_variable, value)                      \
2853   u32 direction_variable = (u32)(value) >> 31;                                 \
2854   if(value == 0)                                                               \
2855     direction_variable = 2                                                     \
2856
2857 #define triangle_case(direction_a, direction_b, direction_c, winding)          \
2858   case (triangle_y_direction_##direction_a |                                   \
2859    (triangle_y_direction_##direction_b << 2) |                                 \
2860    (triangle_y_direction_##direction_c << 4) |                                 \
2861    (triangle_winding_##winding << 6))                                          \
2862
2863 static int prepare_triangle(psx_gpu_struct *psx_gpu, vertex_struct *vertexes,
2864  vertex_struct *vertexes_out[3])
2865 {
2866   s32 y_top, y_bottom;
2867   s32 triangle_area;
2868   u32 triangle_winding = 0;
2869
2870   vertex_struct *a = &(vertexes[0]);
2871   vertex_struct *b = &(vertexes[1]);
2872   vertex_struct *c = &(vertexes[2]);
2873
2874   triangle_area = triangle_signed_area_x2(a->x, a->y, b->x, b->y, c->x, c->y);
2875
2876 #ifdef PROFILE
2877   triangles++;
2878 #endif
2879
2880   if(triangle_area == 0)
2881   {
2882 #ifdef PROFILE
2883     trivial_rejects++;
2884 #endif
2885     return 0;
2886   }
2887
2888   if(b->y < a->y)
2889     vertex_swap(a, b);
2890
2891   if(c->y < b->y)
2892   {
2893     vertex_swap(b, c);
2894
2895     if(b->y < a->y)
2896       vertex_swap(a, b);
2897   }
2898
2899   y_bottom = c->y;
2900   y_top = a->y;
2901
2902   if((y_bottom - y_top) >= 512)
2903   {
2904 #ifdef PROFILE
2905     trivial_rejects++;
2906 #endif
2907     return 0;
2908   }
2909
2910   if(triangle_area < 0)
2911   {
2912     triangle_area = -triangle_area;
2913     triangle_winding ^= 1;
2914     vertex_swap(a, c);
2915   }
2916
2917   if(b->x < a->x)
2918     vertex_swap(a, b);
2919
2920   if(c->x < b->x)
2921   {
2922     vertex_swap(b, c);
2923
2924     if(b->x < a->x)
2925       vertex_swap(a, b);
2926   }
2927
2928   if((c->x - psx_gpu->offset_x) >= 1024 || (c->x - a->x) >= 1024)
2929   {
2930 #ifdef PROFILE
2931     trivial_rejects++;
2932 #endif
2933     return 0;
2934   }
2935
2936   if(invalidate_texture_cache_region_viewport(psx_gpu, a->x, y_top, c->x,
2937    y_bottom) == 0)
2938   {
2939 #ifdef PROFILE
2940     trivial_rejects++;
2941 #endif
2942     return 0;
2943   }
2944
2945   psx_gpu->triangle_area = triangle_area;
2946   psx_gpu->triangle_winding = triangle_winding;
2947
2948   vertexes_out[0] = a;
2949   vertexes_out[1] = b;
2950   vertexes_out[2] = c;
2951
2952   return 1;
2953 }
2954
2955 static void render_triangle_p(psx_gpu_struct *psx_gpu,
2956  vertex_struct *vertex_ptrs[3], u32 flags)
2957 {
2958   psx_gpu->num_spans = 0;
2959
2960   vertex_struct *a = vertex_ptrs[0];
2961   vertex_struct *b = vertex_ptrs[1];
2962   vertex_struct *c = vertex_ptrs[2];
2963
2964   s32 y_delta_a = b->y - a->y;
2965   s32 y_delta_b = c->y - b->y;
2966   s32 y_delta_c = c->y - a->y;
2967
2968   triangle_set_direction(y_direction_a, y_delta_a);
2969   triangle_set_direction(y_direction_b, y_delta_b);
2970   triangle_set_direction(y_direction_c, y_delta_c);
2971
2972   compute_all_gradients(psx_gpu, a, b, c);
2973
2974   switch(y_direction_a | (y_direction_b << 2) | (y_direction_c << 4) |
2975    (psx_gpu->triangle_winding << 6))
2976   {
2977     triangle_case(up, up, up, negative):
2978     triangle_case(up, up, flat, negative):
2979     triangle_case(up, up, down, negative):
2980       setup_spans_up_right(psx_gpu, a, b, c);
2981       break;
2982
2983     triangle_case(flat, up, up, negative):
2984     triangle_case(flat, up, flat, negative):
2985     triangle_case(flat, up, down, negative):
2986       setup_spans_up_a(psx_gpu, a, b, c);
2987       break;
2988
2989     triangle_case(down, up, up, negative):
2990       setup_spans_up_down(psx_gpu, a, c, b);
2991       break;
2992
2993     triangle_case(down, up, flat, negative):
2994       setup_spans_down_a(psx_gpu, a, c, b);
2995       break;
2996
2997     triangle_case(down, up, down, negative):
2998       setup_spans_down_right(psx_gpu, a, c, b);
2999       break;
3000
3001     triangle_case(down, flat, up, negative):
3002     triangle_case(down, flat, flat, negative):
3003     triangle_case(down, flat, down, negative):
3004       setup_spans_down_b(psx_gpu, a, b, c);
3005       break;
3006
3007     triangle_case(down, down, up, negative):
3008     triangle_case(down, down, flat, negative):
3009     triangle_case(down, down, down, negative):
3010       setup_spans_down_left(psx_gpu, a, b, c);
3011       break;
3012
3013     triangle_case(up, up, up, positive):
3014     triangle_case(up, up, flat, positive):
3015     triangle_case(up, up, down, positive):
3016       setup_spans_up_left(psx_gpu, a, b, c);
3017       break;
3018
3019     triangle_case(up, flat, up, positive):
3020     triangle_case(up, flat, flat, positive):
3021     triangle_case(up, flat, down, positive):
3022       setup_spans_up_b(psx_gpu, a, b, c);
3023       break;
3024
3025     triangle_case(up, down, up, positive):
3026       setup_spans_up_right(psx_gpu, a, c, b);
3027       break;
3028
3029     triangle_case(up, down, flat, positive):
3030       setup_spans_up_a(psx_gpu, a, c, b);
3031       break;
3032
3033     triangle_case(up, down, down, positive):
3034       setup_spans_up_down(psx_gpu, a, b, c);
3035       break;
3036
3037     triangle_case(flat, down, up, positive):
3038     triangle_case(flat, down, flat, positive):
3039     triangle_case(flat, down, down, positive):
3040       setup_spans_down_a(psx_gpu, a, b, c);
3041       break;
3042
3043     triangle_case(down, down, up, positive):
3044     triangle_case(down, down, flat, positive):
3045     triangle_case(down, down, down, positive):
3046       setup_spans_down_right(psx_gpu, a, b, c);
3047       break;
3048   }
3049
3050 #ifdef PROFILE
3051   spans += psx_gpu->num_spans;
3052 #endif
3053
3054   if(unlikely(psx_gpu->render_mode & RENDER_INTERLACE_ENABLED))
3055   {
3056     u32 i;
3057
3058     if(psx_gpu->render_mode & RENDER_INTERLACE_ODD)
3059     {
3060       for(i = 0; i < psx_gpu->num_spans; i++)
3061       {
3062         if((psx_gpu->span_edge_data[i].y & 1) == 0)
3063           psx_gpu->span_edge_data[i].num_blocks = 0;
3064       }
3065     }
3066     else
3067     {
3068       for(i = 0; i < psx_gpu->num_spans; i++)
3069       {
3070         if(psx_gpu->span_edge_data[i].y & 1)
3071           psx_gpu->span_edge_data[i].num_blocks = 0;
3072       }
3073     }
3074   }
3075   assert(psx_gpu->span_edge_data[0].y < 1024u);
3076
3077   u32 render_state = flags &
3078    (RENDER_FLAGS_MODULATE_TEXELS | RENDER_FLAGS_BLEND |
3079    RENDER_FLAGS_TEXTURE_MAP | RENDER_FLAGS_SHADE);
3080   render_state |= psx_gpu->render_state_base;
3081
3082   if((psx_gpu->render_state != render_state) ||
3083    (psx_gpu->primitive_type != PRIMITIVE_TYPE_TRIANGLE))
3084   {
3085     psx_gpu->render_state = render_state;
3086     flush_render_block_buffer(psx_gpu);
3087 #ifdef PROFILE
3088     state_changes++;
3089 #endif
3090   }
3091
3092   psx_gpu->primitive_type = PRIMITIVE_TYPE_TRIANGLE;
3093
3094   psx_gpu->render_block_handler =
3095    &(render_triangle_block_handlers[render_state]);
3096   ((setup_blocks_function_type *)psx_gpu->render_block_handler->setup_blocks)
3097    (psx_gpu);
3098 }
3099
3100 void render_triangle(psx_gpu_struct *psx_gpu, vertex_struct *vertexes,
3101  u32 flags)
3102 {
3103   vertex_struct *vertex_ptrs[3];
3104   if (prepare_triangle(psx_gpu, vertexes, vertex_ptrs))
3105     render_triangle_p(psx_gpu, vertex_ptrs, flags);
3106 }
3107
3108 #if !defined(NEON_BUILD) || defined(SIMD_BUILD)
3109
3110 void texture_sprite_blocks_8bpp(psx_gpu_struct *psx_gpu)
3111 {
3112   block_struct *block = psx_gpu->blocks;
3113   u32 num_blocks = psx_gpu->num_blocks;
3114
3115   vec_8x16u texels;
3116   vec_8x8u texel_indexes;
3117
3118   u16 *clut_ptr = psx_gpu->clut_ptr;
3119   u32 i;
3120
3121   while(num_blocks)
3122   {
3123     texel_indexes = block->r;
3124
3125     for(i = 0; i < 8; i++)
3126     {
3127       texels.e[i] = clut_ptr[texel_indexes.e[i]];
3128     }
3129
3130     block->texels = texels;
3131
3132     num_blocks--;
3133     block++;
3134   }
3135 }
3136
3137 #endif
3138
3139
3140 #define setup_sprite_tiled_initialize_4bpp_clut()                              \
3141   u16 *clut_ptr = psx_gpu->clut_ptr;                                           \
3142   vec_8x16u clut_a, clut_b;                                                    \
3143   vec_16x8u clut_low, clut_high;                                               \
3144                                                                                \
3145   load_8x16b(clut_a, clut_ptr);                                                \
3146   load_8x16b(clut_b, clut_ptr + 8);                                            \
3147   unzip_16x8b(clut_low, clut_high, clut_a, clut_b)                             \
3148
3149 #define setup_sprite_tiled_initialize_4bpp()                                   \
3150   setup_sprite_tiled_initialize_4bpp_clut();                                   \
3151                                                                                \
3152   if(psx_gpu->current_texture_mask & psx_gpu->dirty_textures_4bpp_mask)        \
3153     update_texture_4bpp_cache(psx_gpu)                                         \
3154
3155 #define setup_sprite_tiled_initialize_8bpp()                                   \
3156   if(psx_gpu->current_texture_mask & psx_gpu->dirty_textures_8bpp_mask)        \
3157     update_texture_8bpp_cache(psx_gpu)                                         \
3158
3159
3160 #define setup_sprite_tile_fetch_texel_block_8bpp(offset)                       \
3161   texture_block_ptr = (u8 *)psx_gpu->texture_page_ptr +                        \
3162    ((texture_offset + offset) & texture_mask);                                 \
3163                                                                                \
3164   load_64b(texels, texture_block_ptr)                                          \
3165
3166
3167 #define setup_sprite_tile_add_blocks(tile_num_blocks)                          \
3168   num_blocks += tile_num_blocks;                                               \
3169   sprite_blocks += tile_num_blocks;                                            \
3170                                                                                \
3171   if(num_blocks > MAX_BLOCKS)                                                  \
3172   {                                                                            \
3173     flush_render_block_buffer(psx_gpu);                                        \
3174     num_blocks = tile_num_blocks;                                              \
3175     block = psx_gpu->blocks;                                                   \
3176   }                                                                            \
3177
3178 #define setup_sprite_tile_full_4bpp(edge)                                      \
3179 {                                                                              \
3180   vec_8x8u texels_low, texels_high;                                            \
3181   vec_8x16u pixels;                                                            \
3182   setup_sprite_tile_add_blocks(sub_tile_height * 2);                           \
3183                                                                                \
3184   while(sub_tile_height)                                                       \
3185   {                                                                            \
3186     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3187     tbl_16(texels_low, texels, clut_low);                                      \
3188     tbl_16(texels_high, texels, clut_high);                                    \
3189     zip_8x16b(pixels, texels_low, texels_high);                                \
3190                                                                                \
3191     block->texels = pixels;                                                    \
3192     block->draw_mask_bits = left_mask_bits;                                    \
3193     block->fb_ptr = fb_ptr;                                                    \
3194     block++;                                                                   \
3195                                                                                \
3196     setup_sprite_tile_fetch_texel_block_8bpp(8);                               \
3197     tbl_16(texels_low, texels, clut_low);                                      \
3198     tbl_16(texels_high, texels, clut_high);                                    \
3199     zip_8x16b(pixels, texels_low, texels_high);                                \
3200                                                                                \
3201     block->texels = pixels;                                                    \
3202     block->draw_mask_bits = right_mask_bits;                                   \
3203     block->fb_ptr = fb_ptr + 8;                                                \
3204     block++;                                                                   \
3205                                                                                \
3206     fb_ptr += 1024;                                                            \
3207     texture_offset += 0x10;                                                    \
3208     sub_tile_height--;                                                         \
3209   }                                                                            \
3210   texture_offset += 0xF00;                                                     \
3211   psx_gpu->num_blocks = num_blocks;                                            \
3212 }                                                                              \
3213
3214 #define setup_sprite_tile_half_4bpp(edge)                                      \
3215 {                                                                              \
3216   vec_8x8u texels_low, texels_high;                                            \
3217   vec_8x16u pixels;                                                            \
3218   setup_sprite_tile_add_blocks(sub_tile_height);                               \
3219                                                                                \
3220   while(sub_tile_height)                                                       \
3221   {                                                                            \
3222     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3223     tbl_16(texels_low, texels, clut_low);                                      \
3224     tbl_16(texels_high, texels, clut_high);                                    \
3225     zip_8x16b(pixels, texels_low, texels_high);                                \
3226                                                                                \
3227     block->texels = pixels;                                                    \
3228     block->draw_mask_bits = edge##_mask_bits;                                  \
3229     block->fb_ptr = fb_ptr;                                                    \
3230     block++;                                                                   \
3231                                                                                \
3232     fb_ptr += 1024;                                                            \
3233     texture_offset += 0x10;                                                    \
3234     sub_tile_height--;                                                         \
3235   }                                                                            \
3236   texture_offset += 0xF00;                                                     \
3237   psx_gpu->num_blocks = num_blocks;                                            \
3238 }                                                                              \
3239
3240
3241 #define setup_sprite_tile_full_8bpp(edge)                                      \
3242 {                                                                              \
3243   setup_sprite_tile_add_blocks(sub_tile_height * 2);                           \
3244                                                                                \
3245   while(sub_tile_height)                                                       \
3246   {                                                                            \
3247     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3248     block->r = texels;                                                         \
3249     block->draw_mask_bits = left_mask_bits;                                    \
3250     block->fb_ptr = fb_ptr;                                                    \
3251     block++;                                                                   \
3252                                                                                \
3253     setup_sprite_tile_fetch_texel_block_8bpp(8);                               \
3254     block->r = texels;                                                         \
3255     block->draw_mask_bits = right_mask_bits;                                   \
3256     block->fb_ptr = fb_ptr + 8;                                                \
3257     block++;                                                                   \
3258                                                                                \
3259     fb_ptr += 1024;                                                            \
3260     texture_offset += 0x10;                                                    \
3261     sub_tile_height--;                                                         \
3262   }                                                                            \
3263   texture_offset += 0xF00;                                                     \
3264   psx_gpu->num_blocks = num_blocks;                                            \
3265 }                                                                              \
3266
3267 #define setup_sprite_tile_half_8bpp(edge)                                      \
3268 {                                                                              \
3269   setup_sprite_tile_add_blocks(sub_tile_height);                               \
3270                                                                                \
3271   while(sub_tile_height)                                                       \
3272   {                                                                            \
3273     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3274     block->r = texels;                                                         \
3275     block->draw_mask_bits = edge##_mask_bits;                                  \
3276     block->fb_ptr = fb_ptr;                                                    \
3277     block++;                                                                   \
3278                                                                                \
3279     fb_ptr += 1024;                                                            \
3280     texture_offset += 0x10;                                                    \
3281     sub_tile_height--;                                                         \
3282   }                                                                            \
3283   texture_offset += 0xF00;                                                     \
3284   psx_gpu->num_blocks = num_blocks;                                            \
3285 }                                                                              \
3286
3287
3288 #define setup_sprite_tile_column_edge_pre_adjust_half_right()                  \
3289   texture_offset = texture_offset_base + 8;                                    \
3290   fb_ptr += 8                                                                  \
3291
3292 #define setup_sprite_tile_column_edge_pre_adjust_half_left()                   \
3293   texture_offset = texture_offset_base                                         \
3294
3295 #define setup_sprite_tile_column_edge_pre_adjust_half(edge)                    \
3296   setup_sprite_tile_column_edge_pre_adjust_half_##edge()                       \
3297
3298 #define setup_sprite_tile_column_edge_pre_adjust_full(edge)                    \
3299   texture_offset = texture_offset_base                                         \
3300
3301 #define setup_sprite_tile_column_edge_post_adjust_half_right()                 \
3302   fb_ptr -= 8                                                                  \
3303
3304 #define setup_sprite_tile_column_edge_post_adjust_half_left()                  \
3305
3306 #define setup_sprite_tile_column_edge_post_adjust_half(edge)                   \
3307   setup_sprite_tile_column_edge_post_adjust_half_##edge()                      \
3308
3309 #define setup_sprite_tile_column_edge_post_adjust_full(edge)                   \
3310
3311
3312 #define setup_sprite_tile_column_height_single(edge_mode, edge, texture_mode,  \
3313  x4mode)                                                                       \
3314 do                                                                             \
3315 {                                                                              \
3316   sub_tile_height = column_data;                                               \
3317   setup_sprite_tile_column_edge_pre_adjust_##edge_mode##x4mode(edge);          \
3318   setup_sprite_tile_##edge_mode##_##texture_mode##x4mode(edge);                \
3319   setup_sprite_tile_column_edge_post_adjust_##edge_mode##x4mode(edge);         \
3320 } while(0)                                                                     \
3321
3322 #define setup_sprite_tile_column_height_multi(edge_mode, edge, texture_mode,   \
3323  x4mode)                                                                       \
3324 do                                                                             \
3325 {                                                                              \
3326   u32 tiles_remaining = column_data >> 16;                                     \
3327   sub_tile_height = column_data & 0xFF;                                        \
3328   setup_sprite_tile_column_edge_pre_adjust_##edge_mode##x4mode(edge);          \
3329   setup_sprite_tile_##edge_mode##_##texture_mode##x4mode(edge);                \
3330   tiles_remaining -= 1;                                                        \
3331                                                                                \
3332   while(tiles_remaining)                                                       \
3333   {                                                                            \
3334     sub_tile_height = 16;                                                      \
3335     setup_sprite_tile_##edge_mode##_##texture_mode##x4mode(edge);              \
3336     tiles_remaining--;                                                         \
3337   }                                                                            \
3338                                                                                \
3339   sub_tile_height = (column_data >> 8) & 0xFF;                                 \
3340   setup_sprite_tile_##edge_mode##_##texture_mode##x4mode(edge);                \
3341   setup_sprite_tile_column_edge_post_adjust_##edge_mode##x4mode(edge);         \
3342 } while(0)                                                                     \
3343
3344
3345 #define setup_sprite_column_data_single()                                      \
3346   column_data = height                                                         \
3347
3348 #define setup_sprite_column_data_multi()                                       \
3349   column_data = 16 - offset_v;                                                 \
3350   column_data |= ((height_rounded & 0xF) + 1) << 8;                            \
3351   column_data |= (tile_height - 1) << 16                                       \
3352
3353
3354 #define RIGHT_MASK_BIT_SHIFT 8
3355 #define RIGHT_MASK_BIT_SHIFT_4x 16
3356
3357 #define setup_sprite_tile_column_width_single(texture_mode, multi_height,      \
3358  edge_mode, edge, x4mode)                                                      \
3359 {                                                                              \
3360   setup_sprite_column_data_##multi_height();                                   \
3361   left_mask_bits = left_block_mask | right_block_mask;                         \
3362   right_mask_bits = left_mask_bits >> RIGHT_MASK_BIT_SHIFT##x4mode;            \
3363                                                                                \
3364   setup_sprite_tile_column_height_##multi_height(edge_mode, edge,              \
3365    texture_mode, x4mode);                                                      \
3366 }                                                                              \
3367
3368 #define setup_sprite_tiled_advance_column()                                    \
3369   texture_offset_base += 0x100;                                                \
3370   if((texture_offset_base & 0xF00) == 0)                                       \
3371     texture_offset_base -= (0x100 + 0xF00)                                     \
3372
3373 #define FB_PTR_MULTIPLIER 1
3374 #define FB_PTR_MULTIPLIER_4x 2
3375
3376 #define setup_sprite_tile_column_width_multi(texture_mode, multi_height,       \
3377  left_mode, right_mode, x4mode)                                                \
3378 {                                                                              \
3379   setup_sprite_column_data_##multi_height();                                   \
3380   s32 fb_ptr_advance_column = (16 - (1024 * height))                           \
3381     * FB_PTR_MULTIPLIER##x4mode;                                               \
3382                                                                                \
3383   tile_width -= 2;                                                             \
3384   left_mask_bits = left_block_mask;                                            \
3385   right_mask_bits = left_mask_bits >> RIGHT_MASK_BIT_SHIFT##x4mode;            \
3386                                                                                \
3387   setup_sprite_tile_column_height_##multi_height(left_mode, right,             \
3388    texture_mode, x4mode);                                                      \
3389   fb_ptr += fb_ptr_advance_column;                                             \
3390                                                                                \
3391   left_mask_bits = 0x00;                                                       \
3392   right_mask_bits = 0x00;                                                      \
3393                                                                                \
3394   while(tile_width)                                                            \
3395   {                                                                            \
3396     setup_sprite_tiled_advance_column();                                       \
3397     setup_sprite_tile_column_height_##multi_height(full, none,                 \
3398      texture_mode, x4mode);                                                    \
3399     fb_ptr += fb_ptr_advance_column;                                           \
3400     tile_width--;                                                              \
3401   }                                                                            \
3402                                                                                \
3403   left_mask_bits = right_block_mask;                                           \
3404   right_mask_bits = left_mask_bits >> RIGHT_MASK_BIT_SHIFT##x4mode;            \
3405                                                                                \
3406   setup_sprite_tiled_advance_column();                                         \
3407   setup_sprite_tile_column_height_##multi_height(right_mode, left,             \
3408    texture_mode, x4mode);                                                      \
3409 }                                                                              \
3410
3411
3412 /* 4x stuff */
3413 #define setup_sprite_tiled_initialize_4bpp_4x()                                \
3414   setup_sprite_tiled_initialize_4bpp_clut()                                    \
3415
3416 #define setup_sprite_tiled_initialize_8bpp_4x()                                \
3417
3418
3419 #define setup_sprite_tile_full_4bpp_4x(edge)                                   \
3420 {                                                                              \
3421   vec_8x8u texels_low, texels_high;                                            \
3422   vec_8x16u pixels, pixels_wide;                                               \
3423   setup_sprite_tile_add_blocks(sub_tile_height * 2 * 4);                       \
3424   u32 left_mask_bits_a = left_mask_bits & 0xFF;                                \
3425   u32 left_mask_bits_b = left_mask_bits >> 8;                                  \
3426   u32 right_mask_bits_a = right_mask_bits & 0xFF;                              \
3427   u32 right_mask_bits_b = right_mask_bits >> 8;                                \
3428                                                                                \
3429   while(sub_tile_height)                                                       \
3430   {                                                                            \
3431     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3432     tbl_16(texels_low, texels, clut_low);                                      \
3433     tbl_16(texels_high, texels, clut_high);                                    \
3434     zip_8x16b(pixels, texels_low, texels_high);                                \
3435                                                                                \
3436     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.low, pixels.low);    \
3437     block->texels = pixels_wide;                                               \
3438     block->draw_mask_bits = left_mask_bits_a;                                  \
3439     block->fb_ptr = fb_ptr;                                                    \
3440     block++;                                                                   \
3441                                                                                \
3442     block->texels = pixels_wide;                                               \
3443     block->draw_mask_bits = left_mask_bits_a;                                  \
3444     block->fb_ptr = fb_ptr + 1024;                                             \
3445     block++;                                                                   \
3446                                                                                \
3447     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.high, pixels.high);  \
3448     block->texels = pixels_wide;                                               \
3449     block->draw_mask_bits = left_mask_bits_b;                                  \
3450     block->fb_ptr = fb_ptr + 8;                                                \
3451     block++;                                                                   \
3452                                                                                \
3453     block->texels = pixels_wide;                                               \
3454     block->draw_mask_bits = left_mask_bits_b;                                  \
3455     block->fb_ptr = fb_ptr + 1024 + 8;                                         \
3456     block++;                                                                   \
3457                                                                                \
3458     setup_sprite_tile_fetch_texel_block_8bpp(8);                               \
3459     tbl_16(texels_low, texels, clut_low);                                      \
3460     tbl_16(texels_high, texels, clut_high);                                    \
3461     zip_8x16b(pixels, texels_low, texels_high);                                \
3462                                                                                \
3463     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.low, pixels.low);    \
3464     block->texels = pixels_wide;                                               \
3465     block->draw_mask_bits = right_mask_bits_a;                                 \
3466     block->fb_ptr = fb_ptr + 16;                                               \
3467     block++;                                                                   \
3468                                                                                \
3469     block->texels = pixels_wide;                                               \
3470     block->draw_mask_bits = right_mask_bits_a;                                 \
3471     block->fb_ptr = fb_ptr + 1024 + 16;                                        \
3472     block++;                                                                   \
3473                                                                                \
3474     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.high, pixels.high);  \
3475     block->texels = pixels_wide;                                               \
3476     block->draw_mask_bits = right_mask_bits_b;                                 \
3477     block->fb_ptr = fb_ptr + 24;                                               \
3478     block++;                                                                   \
3479                                                                                \
3480     block->texels = pixels_wide;                                               \
3481     block->draw_mask_bits = right_mask_bits_b;                                 \
3482     block->fb_ptr = fb_ptr + 1024 + 24;                                        \
3483     block++;                                                                   \
3484                                                                                \
3485     fb_ptr += 2048;                                                            \
3486     texture_offset += 0x10;                                                    \
3487     sub_tile_height--;                                                         \
3488   }                                                                            \
3489   texture_offset += 0xF00;                                                     \
3490   psx_gpu->num_blocks = num_blocks;                                            \
3491 }                                                                              \
3492
3493 #define setup_sprite_tile_half_4bpp_4x(edge)                                   \
3494 {                                                                              \
3495   vec_8x8u texels_low, texels_high;                                            \
3496   vec_8x16u pixels, pixels_wide;                                               \
3497   setup_sprite_tile_add_blocks(sub_tile_height * 4);                           \
3498   u32 edge##_mask_bits_a = edge##_mask_bits & 0xFF;                            \
3499   u32 edge##_mask_bits_b = edge##_mask_bits >> 8;                              \
3500                                                                                \
3501   while(sub_tile_height)                                                       \
3502   {                                                                            \
3503     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3504     tbl_16(texels_low, texels, clut_low);                                      \
3505     tbl_16(texels_high, texels, clut_high);                                    \
3506     zip_8x16b(pixels, texels_low, texels_high);                                \
3507                                                                                \
3508     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.low, pixels.low);    \
3509     block->texels = pixels_wide;                                               \
3510     block->draw_mask_bits = edge##_mask_bits_a;                                \
3511     block->fb_ptr = fb_ptr;                                                    \
3512     block++;                                                                   \
3513                                                                                \
3514     block->texels = pixels_wide;                                               \
3515     block->draw_mask_bits = edge##_mask_bits_a;                                \
3516     block->fb_ptr = fb_ptr + 1024;                                             \
3517     block++;                                                                   \
3518                                                                                \
3519     zip_4x32b(vector_cast(vec_4x32u, pixels_wide), pixels.high, pixels.high);  \
3520     block->texels = pixels_wide;                                               \
3521     block->draw_mask_bits = edge##_mask_bits_b;                                \
3522     block->fb_ptr = fb_ptr + 8;                                                \
3523     block++;                                                                   \
3524                                                                                \
3525     block->texels = pixels_wide;                                               \
3526     block->draw_mask_bits = edge##_mask_bits_b;                                \
3527     block->fb_ptr = fb_ptr + 1024 + 8;                                         \
3528     block++;                                                                   \
3529                                                                                \
3530     fb_ptr += 2048;                                                            \
3531     texture_offset += 0x10;                                                    \
3532     sub_tile_height--;                                                         \
3533   }                                                                            \
3534   texture_offset += 0xF00;                                                     \
3535   psx_gpu->num_blocks = num_blocks;                                            \
3536 }                                                                              \
3537
3538
3539 #define setup_sprite_tile_full_8bpp_4x(edge)                                   \
3540 {                                                                              \
3541   setup_sprite_tile_add_blocks(sub_tile_height * 2 * 4);                       \
3542   vec_16x8u texels_wide;                                                       \
3543   u32 left_mask_bits_a = left_mask_bits & 0xFF;                                \
3544   u32 left_mask_bits_b = left_mask_bits >> 8;                                  \
3545   u32 right_mask_bits_a = right_mask_bits & 0xFF;                              \
3546   u32 right_mask_bits_b = right_mask_bits >> 8;                                \
3547                                                                                \
3548   while(sub_tile_height)                                                       \
3549   {                                                                            \
3550     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3551     zip_8x16b(vector_cast(vec_8x16u, texels_wide), texels, texels);            \
3552     block->r = texels_wide.low;                                                \
3553     block->draw_mask_bits = left_mask_bits_a;                                  \
3554     block->fb_ptr = fb_ptr;                                                    \
3555     block++;                                                                   \
3556                                                                                \
3557     block->r = texels_wide.low;                                                \
3558     block->draw_mask_bits = left_mask_bits_a;                                  \
3559     block->fb_ptr = fb_ptr + 1024;                                             \
3560     block++;                                                                   \
3561                                                                                \
3562     block->r = texels_wide.high;                                               \
3563     block->draw_mask_bits = left_mask_bits_b;                                  \
3564     block->fb_ptr = fb_ptr + 8;                                                \
3565     block++;                                                                   \
3566                                                                                \
3567     block->r = texels_wide.high;                                               \
3568     block->draw_mask_bits = left_mask_bits_b;                                  \
3569     block->fb_ptr = fb_ptr + 1024 + 8;                                         \
3570     block++;                                                                   \
3571                                                                                \
3572     setup_sprite_tile_fetch_texel_block_8bpp(8);                               \
3573     zip_8x16b(vector_cast(vec_8x16u, texels_wide), texels, texels);            \
3574     block->r = texels_wide.low;                                                \
3575     block->draw_mask_bits = right_mask_bits_a;                                 \
3576     block->fb_ptr = fb_ptr + 16;                                               \
3577     block++;                                                                   \
3578                                                                                \
3579     block->r = texels_wide.low;                                                \
3580     block->draw_mask_bits = right_mask_bits_a;                                 \
3581     block->fb_ptr = fb_ptr + 1024 + 16;                                        \
3582     block++;                                                                   \
3583                                                                                \
3584     block->r = texels_wide.high;                                               \
3585     block->draw_mask_bits = right_mask_bits_b;                                 \
3586     block->fb_ptr = fb_ptr + 24;                                               \
3587     block++;                                                                   \
3588                                                                                \
3589     block->r = texels_wide.high;                                               \
3590     block->draw_mask_bits = right_mask_bits_b;                                 \
3591     block->fb_ptr = fb_ptr + 24 + 1024;                                        \
3592     block++;                                                                   \
3593                                                                                \
3594     fb_ptr += 2048;                                                            \
3595     texture_offset += 0x10;                                                    \
3596     sub_tile_height--;                                                         \
3597   }                                                                            \
3598   texture_offset += 0xF00;                                                     \
3599   psx_gpu->num_blocks = num_blocks;                                            \
3600 }                                                                              \
3601
3602 #define setup_sprite_tile_half_8bpp_4x(edge)                                   \
3603 {                                                                              \
3604   setup_sprite_tile_add_blocks(sub_tile_height * 4);                           \
3605   vec_16x8u texels_wide;                                                       \
3606   u32 edge##_mask_bits_a = edge##_mask_bits & 0xFF;                            \
3607   u32 edge##_mask_bits_b = edge##_mask_bits >> 8;                              \
3608                                                                                \
3609   while(sub_tile_height)                                                       \
3610   {                                                                            \
3611     setup_sprite_tile_fetch_texel_block_8bpp(0);                               \
3612     zip_8x16b(vector_cast(vec_8x16u, texels_wide), texels, texels);            \
3613     block->r = texels_wide.low;                                                \
3614     block->draw_mask_bits = edge##_mask_bits_a;                                \
3615     block->fb_ptr = fb_ptr;                                                    \
3616     block++;                                                                   \
3617                                                                                \
3618     block->r = texels_wide.low;                                                \
3619     block->draw_mask_bits = edge##_mask_bits_a;                                \
3620     block->fb_ptr = fb_ptr + 1024;                                             \
3621     block++;                                                                   \
3622                                                                                \
3623     block->r = texels_wide.high;                                               \
3624     block->draw_mask_bits = edge##_mask_bits_b;                                \
3625     block->fb_ptr = fb_ptr + 8;                                                \
3626     block++;                                                                   \
3627                                                                                \
3628     block->r = texels_wide.high;                                               \
3629     block->draw_mask_bits = edge##_mask_bits_b;                                \
3630     block->fb_ptr = fb_ptr + 8 + 1024;                                         \
3631     block++;                                                                   \
3632                                                                                \
3633     fb_ptr += 2048;                                                            \
3634     texture_offset += 0x10;                                                    \
3635     sub_tile_height--;                                                         \
3636   }                                                                            \
3637   texture_offset += 0xF00;                                                     \
3638   psx_gpu->num_blocks = num_blocks;                                            \
3639 }                                                                              \
3640
3641
3642 #define setup_sprite_tile_column_edge_pre_adjust_half_right_4x()               \
3643   texture_offset = texture_offset_base + 8;                                    \
3644   fb_ptr += 16                                                                 \
3645
3646 #define setup_sprite_tile_column_edge_pre_adjust_half_left_4x()                \
3647   texture_offset = texture_offset_base                                         \
3648
3649 #define setup_sprite_tile_column_edge_pre_adjust_half_4x(edge)                 \
3650   setup_sprite_tile_column_edge_pre_adjust_half_##edge##_4x()                  \
3651
3652 #define setup_sprite_tile_column_edge_pre_adjust_full_4x(edge)                 \
3653   texture_offset = texture_offset_base                                         \
3654
3655 #define setup_sprite_tile_column_edge_post_adjust_half_right_4x()              \
3656   fb_ptr -= 16                                                                 \
3657
3658 #define setup_sprite_tile_column_edge_post_adjust_half_left_4x()               \
3659
3660 #define setup_sprite_tile_column_edge_post_adjust_half_4x(edge)                \
3661   setup_sprite_tile_column_edge_post_adjust_half_##edge##_4x()                 \
3662
3663 #define setup_sprite_tile_column_edge_post_adjust_full_4x(edge)                \
3664
3665
3666 #define setup_sprite_offset_u_adjust()                                         \
3667
3668 #define setup_sprite_comapre_left_block_mask()                                 \
3669   ((left_block_mask & 0xFF) == 0xFF)                                           \
3670
3671 #define setup_sprite_comapre_right_block_mask()                                \
3672   (((right_block_mask >> 8) & 0xFF) == 0xFF)                                   \
3673
3674
3675 #define setup_sprite_offset_u_adjust_4x()                                      \
3676   offset_u *= 2;                                                               \
3677   offset_u_right = offset_u_right * 2 + 1                                      \
3678
3679 #define setup_sprite_comapre_left_block_mask_4x()                              \
3680   ((left_block_mask & 0xFFFF) == 0xFFFF)                                       \
3681
3682 #define setup_sprite_comapre_right_block_mask_4x()                             \
3683   (((right_block_mask >> 16) & 0xFFFF) == 0xFFFF)                              \
3684
3685
3686 #define setup_sprite_tiled_builder(texture_mode, x4mode)                       \
3687 void setup_sprite_##texture_mode##x4mode(psx_gpu_struct *psx_gpu, s32 x, s32 y,\
3688  s32 u, s32 v, s32 width, s32 height, u32 color)                               \
3689 {                                                                              \
3690   s32 offset_u = u & 0xF;                                                      \
3691   s32 offset_v = v & 0xF;                                                      \
3692                                                                                \
3693   s32 width_rounded = offset_u + width + 15;                                   \
3694   s32 height_rounded = offset_v + height + 15;                                 \
3695   s32 tile_height = height_rounded / 16;                                       \
3696   s32 tile_width = width_rounded / 16;                                         \
3697   u32 offset_u_right = width_rounded & 0xF;                                    \
3698                                                                                \
3699   setup_sprite_offset_u_adjust##x4mode();                                      \
3700                                                                                \
3701   u32 left_block_mask = ~(0xFFFFFFFF << offset_u);                             \
3702   u32 right_block_mask = 0xFFFFFFFE << offset_u_right;                         \
3703                                                                                \
3704   u32 left_mask_bits;                                                          \
3705   u32 right_mask_bits;                                                         \
3706                                                                                \
3707   u32 sub_tile_height;                                                         \
3708   u32 column_data;                                                             \
3709                                                                                \
3710   u32 texture_mask = (psx_gpu->texture_mask_width & 0xF) |                     \
3711    ((psx_gpu->texture_mask_height & 0xF) << 4) |                               \
3712    ((psx_gpu->texture_mask_width >> 4) << 8) |                                 \
3713    ((psx_gpu->texture_mask_height >> 4) << 12);                                \
3714   u32 texture_offset = ((v & 0xF) << 4) | ((u & 0xF0) << 4) |                  \
3715    ((v & 0xF0) << 8);                                                          \
3716   u32 texture_offset_base = texture_offset;                                    \
3717   u32 control_mask;                                                            \
3718                                                                                \
3719   u16 *fb_ptr = psx_gpu->vram_out_ptr + (y * 1024) + (x - offset_u);           \
3720   u32 num_blocks = psx_gpu->num_blocks;                                        \
3721   block_struct *block = psx_gpu->blocks + num_blocks;                          \
3722                                                                                \
3723   u8 *texture_block_ptr;                                                       \
3724   vec_8x8u texels;                                                             \
3725                                                                                \
3726   setup_sprite_tiled_initialize_##texture_mode##x4mode();                      \
3727                                                                                \
3728   control_mask = tile_width == 1;                                              \
3729   control_mask |= (tile_height == 1) << 1;                                     \
3730   control_mask |= setup_sprite_comapre_left_block_mask##x4mode() << 2;         \
3731   control_mask |= setup_sprite_comapre_right_block_mask##x4mode() << 3;        \
3732                                                                                \
3733   sprites_##texture_mode++;                                                    \
3734                                                                                \
3735   switch(control_mask)                                                         \
3736   {                                                                            \
3737     default:                                                                   \
3738     case 0x0:                                                                  \
3739       setup_sprite_tile_column_width_multi(texture_mode, multi, full, full,    \
3740        x4mode);                                                                \
3741       break;                                                                   \
3742                                                                                \
3743     case 0x1:                                                                  \
3744       setup_sprite_tile_column_width_single(texture_mode, multi, full, none,   \
3745        x4mode);                                                                \
3746       break;                                                                   \
3747                                                                                \
3748     case 0x2:                                                                  \
3749       setup_sprite_tile_column_width_multi(texture_mode, single, full, full,   \
3750        x4mode);                                                                \
3751       break;                                                                   \
3752                                                                                \
3753     case 0x3:                                                                  \
3754       setup_sprite_tile_column_width_single(texture_mode, single, full, none,  \
3755        x4mode);                                                                \
3756       break;                                                                   \
3757                                                                                \
3758     case 0x4:                                                                  \
3759       setup_sprite_tile_column_width_multi(texture_mode, multi, half, full,    \
3760        x4mode);                                                                \
3761       break;                                                                   \
3762                                                                                \
3763     case 0x5:                                                                  \
3764       setup_sprite_tile_column_width_single(texture_mode, multi, half, right,  \
3765        x4mode);                                                                \
3766       break;                                                                   \
3767                                                                                \
3768     case 0x6:                                                                  \
3769       setup_sprite_tile_column_width_multi(texture_mode, single, half, full,   \
3770        x4mode);                                                                \
3771       break;                                                                   \
3772                                                                                \
3773     case 0x7:                                                                  \
3774       setup_sprite_tile_column_width_single(texture_mode, single, half, right, \
3775        x4mode);                                                                \
3776       break;                                                                   \
3777                                                                                \
3778     case 0x8:                                                                  \
3779       setup_sprite_tile_column_width_multi(texture_mode, multi, full, half,    \
3780        x4mode);                                                                \
3781       break;                                                                   \
3782                                                                                \
3783     case 0x9:                                                                  \
3784       setup_sprite_tile_column_width_single(texture_mode, multi, half, left,   \
3785        x4mode);                                                                \
3786       break;                                                                   \
3787                                                                                \
3788     case 0xA:                                                                  \
3789       setup_sprite_tile_column_width_multi(texture_mode, single, full, half,   \
3790        x4mode);                                                                \
3791       break;                                                                   \
3792                                                                                \
3793     case 0xB:                                                                  \
3794       setup_sprite_tile_column_width_single(texture_mode, single, half, left,  \
3795        x4mode);                                                                \
3796       break;                                                                   \
3797                                                                                \
3798     case 0xC:                                                                  \
3799       setup_sprite_tile_column_width_multi(texture_mode, multi, half, half,    \
3800        x4mode);                                                                \
3801       break;                                                                   \
3802                                                                                \
3803     case 0xE:                                                                  \
3804       setup_sprite_tile_column_width_multi(texture_mode, single, half, half,   \
3805        x4mode);                                                                \
3806       break;                                                                   \
3807   }                                                                            \
3808 }                                                                              \
3809
3810 #ifndef NEON_BUILD
3811 setup_sprite_tiled_builder(4bpp,);
3812 setup_sprite_tiled_builder(8bpp,);
3813
3814 setup_sprite_tiled_builder(4bpp,_4x);
3815 setup_sprite_tiled_builder(8bpp,_4x);
3816 #endif
3817
3818 #if !defined(NEON_BUILD) || defined(SIMD_BUILD)
3819
3820 void setup_sprite_16bpp(psx_gpu_struct *psx_gpu, s32 x, s32 y, s32 u,
3821  s32 v, s32 width, s32 height, u32 color)
3822 {
3823   u32 left_offset = u & 0x7;
3824   u32 width_rounded = width + left_offset + 7;
3825
3826   u16 *fb_ptr = psx_gpu->vram_out_ptr + (y * 1024) + (s32)(x - left_offset);
3827   u32 right_width = width_rounded & 0x7;
3828   u32 block_width = width_rounded / 8;
3829   u32 fb_ptr_pitch = (1024 + 8) - (block_width * 8);
3830
3831   u32 left_mask_bits = ~(0xFF << left_offset);
3832   u32 right_mask_bits = 0xFE << right_width;
3833
3834   u32 texture_offset_base = u + (v * 1024);
3835   u32 texture_mask =
3836    psx_gpu->texture_mask_width | (psx_gpu->texture_mask_height * 1024);
3837
3838   u32 blocks_remaining;
3839   u32 num_blocks = psx_gpu->num_blocks;
3840   block_struct *block = psx_gpu->blocks + num_blocks;
3841
3842   u16 *texture_page_ptr = psx_gpu->texture_page_ptr;
3843   u16 *texture_block_ptr;
3844
3845   texture_offset_base &= ~0x7;
3846
3847   stats_add(sprites_16bpp, 1);
3848
3849   if(block_width == 1)
3850   {
3851     u32 mask_bits = left_mask_bits | right_mask_bits;
3852
3853     while(height)
3854     {
3855       num_blocks++;
3856       sprite_blocks++;
3857
3858       if(num_blocks > MAX_BLOCKS)
3859       {
3860         flush_render_block_buffer(psx_gpu);
3861         num_blocks = 1;
3862         block = psx_gpu->blocks;
3863       }
3864
3865       texture_block_ptr =
3866        texture_page_ptr + (texture_offset_base & texture_mask);
3867
3868       block->texels = *(vec_8x16u *)texture_block_ptr;
3869       block->draw_mask_bits = mask_bits;
3870       block->fb_ptr = fb_ptr;
3871
3872       block++;
3873
3874       texture_offset_base += 1024;
3875       fb_ptr += 1024;
3876
3877       height--;
3878       psx_gpu->num_blocks = num_blocks;
3879     }
3880   }
3881   else
3882   {
3883     u32 texture_offset;
3884
3885     while(height)
3886     {
3887       blocks_remaining = block_width - 2;
3888       num_blocks += block_width;
3889       sprite_blocks += block_width;
3890
3891       if(num_blocks > MAX_BLOCKS)
3892       {
3893         flush_render_block_buffer(psx_gpu);
3894         num_blocks = block_width;
3895         block = psx_gpu->blocks;
3896       }
3897
3898       texture_offset = texture_offset_base;
3899       texture_offset_base += 1024;
3900
3901       texture_block_ptr = texture_page_ptr + (texture_offset & texture_mask);
3902       block->texels = *(vec_8x16u *)texture_block_ptr;
3903
3904       block->draw_mask_bits = left_mask_bits;
3905       block->fb_ptr = fb_ptr;
3906
3907       texture_offset += 8;
3908       fb_ptr += 8;
3909       block++;
3910
3911       while(blocks_remaining)
3912       {
3913         texture_block_ptr = texture_page_ptr + (texture_offset & texture_mask);
3914         block->texels = *(vec_8x16u *)texture_block_ptr;
3915
3916         block->draw_mask_bits = 0;
3917         block->fb_ptr = fb_ptr;
3918
3919         texture_offset += 8;
3920         fb_ptr += 8;
3921         block++;
3922
3923         blocks_remaining--;
3924       }
3925
3926       texture_block_ptr = texture_page_ptr + (texture_offset & texture_mask);
3927       block->texels = *(vec_8x16u *)texture_block_ptr;
3928
3929       block->draw_mask_bits = right_mask_bits;
3930       block->fb_ptr = fb_ptr;
3931
3932       fb_ptr += fb_ptr_pitch;
3933       block++;
3934
3935       height--;
3936       psx_gpu->num_blocks = num_blocks;
3937     }
3938   }
3939 }
3940
3941 #endif
3942
3943 #ifndef NEON_BUILD
3944
3945 void setup_sprite_untextured_512(psx_gpu_struct *psx_gpu, s32 x, s32 y, s32 u,
3946  s32 v, s32 width, s32 height, u32 color)
3947 {
3948   u32 right_width = ((width - 1) & 0x7) + 1;
3949   u32 right_mask_bits = (0xFF << right_width);
3950   u16 *fb_ptr = psx_gpu->vram_out_ptr + (y * 1024) + x;
3951   u32 block_width = (width + 7) / 8;
3952   u32 fb_ptr_pitch = 1024 - ((block_width - 1) * 8);
3953   u32 blocks_remaining;
3954   u32 num_blocks = psx_gpu->num_blocks;
3955   block_struct *block = psx_gpu->blocks + num_blocks;
3956
3957   u32 color_r = color & 0xFF;
3958   u32 color_g = (color >> 8) & 0xFF;
3959   u32 color_b = (color >> 16) & 0xFF;
3960   vec_8x16u colors;
3961   vec_8x16u right_mask;
3962   vec_8x16u test_mask = psx_gpu->test_mask;
3963   vec_8x16u zero_mask;
3964
3965   sprites_untextured++;
3966
3967   color = (color_r >> 3) | ((color_g >> 3) << 5) | ((color_b >> 3) << 10);
3968
3969   dup_8x16b(colors, color);
3970   dup_8x16b(zero_mask, 0x00);
3971   dup_8x16b(right_mask, right_mask_bits);
3972   tst_8x16b(right_mask, right_mask, test_mask);
3973
3974   while(height)
3975   {
3976     blocks_remaining = block_width - 1;
3977     num_blocks += block_width;
3978
3979 #ifdef PROFILE
3980     sprite_blocks += block_width;
3981 #endif
3982
3983     if(num_blocks > MAX_BLOCKS)
3984     {
3985       flush_render_block_buffer(psx_gpu);
3986       num_blocks = block_width;
3987       block = psx_gpu->blocks;
3988     }
3989
3990     while(blocks_remaining)
3991     {
3992       block->pixels = colors;
3993       block->draw_mask = zero_mask;
3994       block->fb_ptr = fb_ptr;
3995
3996       fb_ptr += 8;
3997       block++;
3998       blocks_remaining--;
3999     }
4000
4001     block->pixels = colors;
4002     block->draw_mask = right_mask;
4003     block->fb_ptr = fb_ptr;
4004
4005     block++;
4006     fb_ptr += fb_ptr_pitch;
4007
4008     height--;
4009     psx_gpu->num_blocks = num_blocks;
4010   }
4011 }
4012
4013 #endif
4014
4015 static void __attribute__((noinline))
4016 setup_sprite_untextured_simple(psx_gpu_struct *psx_gpu, s32 x, s32 y, s32 u,
4017  s32 v, s32 width, s32 height, u32 color)
4018 {
4019   u32 r = color & 0xFF;
4020   u32 g = (color >> 8) & 0xFF;
4021   u32 b = (color >> 16) & 0xFF;
4022   u32 color_16bpp = (r >> 3) | ((g >> 3) << 5) | ((b >> 3) << 10) |
4023    psx_gpu->mask_msb;
4024   u32 color_32bpp = color_16bpp | (color_16bpp << 16);
4025
4026   u16 *vram_ptr16 = psx_gpu->vram_out_ptr + x + (y * 1024);
4027   u32 *vram_ptr;
4028
4029   u32 num_width;
4030
4031   if(psx_gpu->num_blocks)
4032   {
4033     flush_render_block_buffer(psx_gpu);
4034   }
4035
4036   while(height)
4037   {
4038     num_width = width;
4039
4040     vram_ptr = (void *)vram_ptr16;
4041     if((uintptr_t)vram_ptr16 & 2)
4042     {
4043       *vram_ptr16 = color_32bpp;
4044       vram_ptr = (void *)(vram_ptr16 + 1);
4045       num_width--;
4046     }
4047
4048     while(num_width >= 4 * 2)
4049     {
4050       vram_ptr[0] = color_32bpp;
4051       vram_ptr[1] = color_32bpp;
4052       vram_ptr[2] = color_32bpp;
4053       vram_ptr[3] = color_32bpp;
4054
4055       vram_ptr += 4;
4056       num_width -= 4 * 2;
4057     }
4058
4059     while(num_width >= 2)
4060     {
4061       *vram_ptr++ = color_32bpp;
4062       num_width -= 2;
4063     }
4064
4065     if(num_width > 0)
4066     {
4067       *(u16 *)vram_ptr = color_32bpp;
4068     }
4069
4070     vram_ptr16 += 1024;
4071     height--;
4072   }
4073 }
4074
4075 void setup_sprite_untextured_512(psx_gpu_struct *psx_gpu, s32 x, s32 y, s32 u,
4076  s32 v, s32 width, s32 height, u32 color);
4077
4078 void setup_sprite_untextured(psx_gpu_struct *psx_gpu, s32 x, s32 y, s32 u,
4079  s32 v, s32 width, s32 height, u32 color)
4080 {
4081   if((psx_gpu->render_state & (RENDER_STATE_MASK_EVALUATE |
4082    RENDER_FLAGS_MODULATE_TEXELS | RENDER_FLAGS_BLEND)) == 0 &&
4083    (psx_gpu->render_mode & RENDER_INTERLACE_ENABLED) == 0)
4084   {
4085     setup_sprite_untextured_simple(psx_gpu, x, y, u, v, width, height, color);
4086     return;
4087   }
4088
4089   while (width > 0)
4090   {
4091     s32 w1 = width > 512 ? 512 : width;
4092     setup_sprite_untextured_512(psx_gpu, x, y, 0, 0, w1, height, color);
4093     x += 512;
4094     width -= 512;
4095   }
4096 }
4097
4098
4099 #define setup_sprite_blocks_switch_textured(texture_mode)                      \
4100   setup_sprite_##texture_mode                                                  \
4101
4102 #define setup_sprite_blocks_switch_untextured(texture_mode)                    \
4103   setup_sprite_untextured                                                      \
4104
4105 #define setup_sprite_blocks_switch(texturing, texture_mode)                    \
4106   setup_sprite_blocks_switch_##texturing(texture_mode)                         \
4107
4108
4109 #define texture_sprite_blocks_switch_4bpp()                                    \
4110   texture_blocks_untextured                                                    \
4111
4112 #define texture_sprite_blocks_switch_8bpp()                                    \
4113   texture_sprite_blocks_8bpp                                                   \
4114
4115 #define texture_sprite_blocks_switch_16bpp()                                   \
4116   texture_blocks_untextured                                                    \
4117
4118 #define texture_sprite_blocks_switch_untextured(texture_mode)                  \
4119   texture_blocks_untextured                                                    \
4120
4121 #define texture_sprite_blocks_switch_textured(texture_mode)                    \
4122   texture_sprite_blocks_switch_##texture_mode()                                \
4123
4124 #define render_sprite_blocks_switch_block_modulation(texture_mode, blend_mode, \
4125  mask_evaluate, shading, dithering, texturing, blending, modulation)           \
4126 {                                                                              \
4127   setup_sprite_blocks_switch(texturing, texture_mode),                         \
4128   texture_sprite_blocks_switch_##texturing(texture_mode),                      \
4129   shade_blocks_switch(unshaded, texturing, modulation, undithered, blending,   \
4130    mask_evaluate),                                                             \
4131   blend_blocks_switch(texturing, blending, blend_mode, mask_evaluate)          \
4132 }                                                                              \
4133
4134 #define render_sprite_blocks_switch_block_blending(texture_mode, blend_mode,   \
4135  mask_evaluate, shading, dithering, texturing, blending)                       \
4136   render_sprite_blocks_switch_block_modulation(texture_mode, blend_mode,       \
4137    mask_evaluate, shading, dithering, texturing, blending, modulated),         \
4138   render_sprite_blocks_switch_block_modulation(texture_mode, blend_mode,       \
4139    mask_evaluate, shading, dithering, texturing, blending, unmodulated)        \
4140
4141 #define render_sprite_blocks_switch_block_texturing(texture_mode, blend_mode,  \
4142  mask_evaluate, shading, dithering, texturing)                                 \
4143   render_sprite_blocks_switch_block_blending(texture_mode, blend_mode,         \
4144    mask_evaluate, shading, dithering, texturing, unblended),                   \
4145   render_sprite_blocks_switch_block_blending(texture_mode, blend_mode,         \
4146    mask_evaluate, shading, dithering, texturing, blended)                      \
4147
4148 #define render_sprite_blocks_switch_block_dithering(texture_mode, blend_mode,  \
4149  mask_evaluate, shading, dithering)                                            \
4150   render_sprite_blocks_switch_block_texturing(texture_mode, blend_mode,        \
4151    mask_evaluate, shading, dithering, untextured),                             \
4152   render_sprite_blocks_switch_block_texturing(texture_mode, blend_mode,        \
4153    mask_evaluate, shading, dithering, textured)                                \
4154
4155 #define render_sprite_blocks_switch_block_shading(texture_mode, blend_mode,    \
4156  mask_evaluate, shading)                                                       \
4157   render_sprite_blocks_switch_block_dithering(texture_mode, blend_mode,        \
4158    mask_evaluate, shading, undithered),                                        \
4159   render_sprite_blocks_switch_block_dithering(texture_mode, blend_mode,        \
4160    mask_evaluate, shading, dithered)                                           \
4161
4162 #define render_sprite_blocks_switch_block_mask_evaluate(texture_mode,          \
4163  blend_mode, mask_evaluate)                                                    \
4164   render_sprite_blocks_switch_block_shading(texture_mode, blend_mode,          \
4165    mask_evaluate, unshaded),                                                   \
4166   render_sprite_blocks_switch_block_shading(texture_mode, blend_mode,          \
4167    mask_evaluate, shaded)                                                      \
4168
4169 #define render_sprite_blocks_switch_block_blend_mode(texture_mode, blend_mode) \
4170   render_sprite_blocks_switch_block_mask_evaluate(texture_mode, blend_mode,    \
4171    off),                                                                       \
4172   render_sprite_blocks_switch_block_mask_evaluate(texture_mode, blend_mode,    \
4173    on)                                                                         \
4174
4175 #define render_sprite_blocks_switch_block_texture_mode(texture_mode)           \
4176   render_sprite_blocks_switch_block_blend_mode(texture_mode, average),         \
4177   render_sprite_blocks_switch_block_blend_mode(texture_mode, add),             \
4178   render_sprite_blocks_switch_block_blend_mode(texture_mode, subtract),        \
4179   render_sprite_blocks_switch_block_blend_mode(texture_mode, add_fourth)       \
4180
4181 #define render_sprite_blocks_switch_block()                                    \
4182   render_sprite_blocks_switch_block_texture_mode(4bpp),                        \
4183   render_sprite_blocks_switch_block_texture_mode(8bpp),                        \
4184   render_sprite_blocks_switch_block_texture_mode(16bpp),                       \
4185   render_sprite_blocks_switch_block_texture_mode(16bpp)                        \
4186
4187
4188 render_block_handler_struct render_sprite_block_handlers[] =
4189 {
4190   render_sprite_blocks_switch_block()
4191 };
4192
4193
4194 void render_sprite(psx_gpu_struct *psx_gpu, s32 x, s32 y, u32 u, u32 v,
4195  s32 width, s32 height, u32 flags, u32 color)
4196 {
4197   s32 x_right = x + width - 1;
4198   s32 y_bottom = y + height - 1;
4199
4200 #ifdef PROFILE
4201   sprites++;
4202 #endif
4203
4204   if(invalidate_texture_cache_region_viewport(psx_gpu, x, y, x_right,
4205    y_bottom) == 0)
4206   {
4207     return;
4208   }
4209
4210   if(x < psx_gpu->viewport_start_x)
4211   {
4212     u32 clip = psx_gpu->viewport_start_x - x;
4213     x += clip;
4214     u += clip;
4215     width -= clip;
4216   }
4217
4218   if(y < psx_gpu->viewport_start_y)
4219   {
4220     s32 clip = psx_gpu->viewport_start_y - y;
4221     y += clip;
4222     v += clip;
4223     height -= clip;
4224   }
4225
4226   if(x_right > psx_gpu->viewport_end_x)
4227     width -= x_right - psx_gpu->viewport_end_x;
4228
4229   if(y_bottom > psx_gpu->viewport_end_y)
4230     height -= y_bottom - psx_gpu->viewport_end_y;
4231
4232   if((width <= 0) || (height <= 0))
4233     return;
4234
4235 #ifdef PROFILE
4236   span_pixels += width * height;
4237   spans += height;
4238 #endif
4239
4240   u32 render_state = flags &
4241    (RENDER_FLAGS_MODULATE_TEXELS | RENDER_FLAGS_BLEND |
4242    RENDER_FLAGS_TEXTURE_MAP);
4243   render_state |=
4244    (psx_gpu->render_state_base & ~RENDER_STATE_DITHER);
4245
4246   if((psx_gpu->render_state != render_state) ||
4247    (psx_gpu->primitive_type != PRIMITIVE_TYPE_SPRITE))
4248   {
4249     psx_gpu->render_state = render_state;
4250     flush_render_block_buffer(psx_gpu);
4251 #ifdef PROFILE
4252     state_changes++;
4253 #endif
4254   }
4255
4256   psx_gpu->primitive_type = PRIMITIVE_TYPE_SPRITE;
4257
4258   color &= 0xFFFFFF;
4259
4260   if(psx_gpu->triangle_color != color)
4261   {
4262     flush_render_block_buffer(psx_gpu);
4263     psx_gpu->triangle_color = color;
4264   }
4265
4266   if(color == 0x808080)
4267     render_state |= RENDER_FLAGS_MODULATE_TEXELS;
4268
4269   render_block_handler_struct *render_block_handler =
4270    &(render_sprite_block_handlers[render_state]);
4271   psx_gpu->render_block_handler = render_block_handler;
4272
4273   ((setup_sprite_function_type *)render_block_handler->setup_blocks)
4274    (psx_gpu, x, y, u, v, width, height, color);
4275 }
4276
4277 #define draw_pixel_line_mask_evaluate_yes()                                    \
4278   if((*vram_ptr & 0x8000) == 0)                                                \
4279
4280 #define draw_pixel_line_mask_evaluate_no()                                     \
4281
4282
4283 #define draw_pixel_line_shaded()                                               \
4284 {                                                                              \
4285   color_r = fixed_to_int(current_r);                                           \
4286   color_g = fixed_to_int(current_g);                                           \
4287   color_b = fixed_to_int(current_b);                                           \
4288                                                                                \
4289   current_r += gradient_r;                                                     \
4290   current_g += gradient_g;                                                     \
4291   current_b += gradient_b;                                                     \
4292 }                                                                              \
4293
4294 #define draw_pixel_line_unshaded()                                             \
4295 {                                                                              \
4296   color_r = color & 0xFF;                                                      \
4297   color_g = (color >> 8) & 0xFF;                                               \
4298   color_b = (color >> 16) & 0xFF;                                              \
4299 }                                                                              \
4300
4301
4302 #define draw_pixel_line_dithered(_x, _y)                                       \
4303 {                                                                              \
4304   u32 dither_xor = _x ^ _y;                                                    \
4305   s32 dither_offset = (dither_xor >> 1) & 0x1;                                 \
4306   dither_offset |= (_y & 0x1) << 1;                                            \
4307   dither_offset |= (dither_xor & 0x1) << 2;                                    \
4308   dither_offset -= 4;                                                          \
4309                                                                                \
4310   color_r += dither_offset;                                                    \
4311   color_g += dither_offset;                                                    \
4312   color_b += dither_offset;                                                    \
4313                                                                                \
4314   if(color_r < 0)                                                              \
4315     color_r = 0;                                                               \
4316                                                                                \
4317   if(color_g < 0)                                                              \
4318     color_g = 0;                                                               \
4319                                                                                \
4320   if(color_b < 0)                                                              \
4321     color_b = 0;                                                               \
4322                                                                                \
4323   if(color_r > 255)                                                            \
4324     color_r = 255;                                                             \
4325                                                                                \
4326   if(color_g > 255)                                                            \
4327     color_g = 255;                                                             \
4328                                                                                \
4329   if(color_b > 255)                                                            \
4330     color_b = 255;                                                             \
4331 }                                                                              \
4332
4333 #define draw_pixel_line_undithered(_x, _y)                                     \
4334
4335
4336 #define draw_pixel_line_average()                                              \
4337   color_r = (color_r + fb_r) / 2;                                              \
4338   color_g = (color_g + fb_g) / 2;                                              \
4339   color_b = (color_b + fb_b) / 2                                               \
4340
4341 #define draw_pixel_line_add()                                                  \
4342   color_r += fb_r;                                                             \
4343   color_g += fb_g;                                                             \
4344   color_b += fb_b;                                                             \
4345                                                                                \
4346   if(color_r > 31)                                                             \
4347     color_r = 31;                                                              \
4348                                                                                \
4349   if(color_g > 31)                                                             \
4350     color_g = 31;                                                              \
4351                                                                                \
4352   if(color_b > 31)                                                             \
4353     color_b = 31                                                               \
4354                                                                                \
4355
4356 #define draw_pixel_line_subtract()                                             \
4357   color_r = fb_r - color_r;                                                    \
4358   color_g = fb_g - color_g;                                                    \
4359   color_b = fb_b - color_b;                                                    \
4360                                                                                \
4361   if(color_r < 0)                                                              \
4362     color_r = 0;                                                               \
4363                                                                                \
4364   if(color_g < 0)                                                              \
4365     color_g = 0;                                                               \
4366                                                                                \
4367   if(color_b < 0)                                                              \
4368     color_b = 0                                                                \
4369
4370 #define draw_pixel_line_add_fourth()                                           \
4371   color_r = fb_r + (color_r / 4);                                              \
4372   color_g = fb_g + (color_g / 4);                                              \
4373   color_b = fb_b + (color_b / 4);                                              \
4374                                                                                \
4375   if(color_r > 31)                                                             \
4376     color_r = 31;                                                              \
4377                                                                                \
4378   if(color_g > 31)                                                             \
4379     color_g = 31;                                                              \
4380                                                                                \
4381   if(color_b > 31)                                                             \
4382     color_b = 31                                                               \
4383
4384
4385 #define draw_pixel_line_blended(blend_mode)                                    \
4386   s32 fb_pixel = *vram_ptr;                                                    \
4387   s32 fb_r = fb_pixel & 0x1F;                                                  \
4388   s32 fb_g = (fb_pixel >> 5) & 0x1F;                                           \
4389   s32 fb_b = (fb_pixel >> 10) & 0x1F;                                          \
4390                                                                                \
4391   draw_pixel_line_##blend_mode()                                               \
4392
4393 #define draw_pixel_line_unblended(blend_mode)                                  \
4394
4395
4396 #define draw_pixel_line(_x, _y, shading, blending, dithering, mask_evaluate,   \
4397  blend_mode)                                                                   \
4398   if((_x >= psx_gpu->viewport_start_x) && (_y >= psx_gpu->viewport_start_y) && \
4399    (_x <= psx_gpu->viewport_end_x) && (_y <= psx_gpu->viewport_end_y))         \
4400   {                                                                            \
4401     draw_pixel_line_mask_evaluate_##mask_evaluate()                            \
4402     {                                                                          \
4403       draw_pixel_line_##shading();                                             \
4404       draw_pixel_line_##dithering(_x, _y);                                     \
4405                                                                                \
4406       color_r >>= 3;                                                           \
4407       color_g >>= 3;                                                           \
4408       color_b >>= 3;                                                           \
4409                                                                                \
4410       draw_pixel_line_##blending(blend_mode);                                  \
4411                                                                                \
4412       *vram_ptr = color_r | (color_g << 5) | (color_b << 10) |                 \
4413        psx_gpu->mask_msb;                                                      \
4414     }                                                                          \
4415   }                                                                            \
4416
4417 #define update_increment(value)                                                \
4418   value++                                                                      \
4419
4420 #define update_decrement(value)                                                \
4421   value--                                                                      \
4422
4423 #define update_vram_row_increment(value)                                       \
4424   vram_ptr += 1024                                                             \
4425
4426 #define update_vram_row_decrement(value)                                       \
4427   vram_ptr -= 1024                                                             \
4428
4429 #define compare_increment(a, b)                                                \
4430   (a <= b)                                                                     \
4431
4432 #define compare_decrement(a, b)                                                \
4433   (a >= b)                                                                     \
4434
4435 #define set_line_gradients(minor)                                              \
4436 {                                                                              \
4437   s32 gradient_divisor = delta_##minor;                                        \
4438   if(gradient_divisor != 0)                                                    \
4439   {                                                                            \
4440     gradient_r = int_to_fixed(vertex_b->r - vertex_a->r) / gradient_divisor;   \
4441     gradient_g = int_to_fixed(vertex_b->g - vertex_a->g) / gradient_divisor;   \
4442     gradient_b = int_to_fixed(vertex_b->b - vertex_a->b) / gradient_divisor;   \
4443   }                                                                            \
4444   else                                                                         \
4445   {                                                                            \
4446     gradient_r = 0;                                                            \
4447     gradient_g = 0;                                                            \
4448     gradient_b = 0;                                                            \
4449   }                                                                            \
4450   current_r = fixed_center(vertex_a->r);                                       \
4451   current_g = fixed_center(vertex_a->g);                                       \
4452   current_b = fixed_center(vertex_a->b);                                       \
4453 }
4454
4455 #define draw_line_span_horizontal(direction, shading, blending, dithering,     \
4456  mask_evaluate, blend_mode)                                                    \
4457 do                                                                             \
4458 {                                                                              \
4459   error_step = delta_y * 2;                                                    \
4460   error_wrap = delta_x * 2;                                                    \
4461   error = delta_x;                                                             \
4462                                                                                \
4463   current_y = y_a;                                                             \
4464   set_line_gradients(x);                                                       \
4465                                                                                \
4466   for(current_x = x_a; current_x <= x_b; current_x++)                          \
4467   {                                                                            \
4468     draw_pixel_line(current_x, current_y, shading, blending, dithering,        \
4469      mask_evaluate, blend_mode);                                               \
4470     error += error_step;                                                       \
4471     vram_ptr++;                                                                \
4472                                                                                \
4473     if(error >= error_wrap)                                                    \
4474     {                                                                          \
4475       update_##direction(current_y);                                           \
4476       update_vram_row_##direction();                                           \
4477       error -= error_wrap;                                                     \
4478     }                                                                          \
4479   }                                                                            \
4480 } while(0)                                                                     \
4481
4482 #define draw_line_span_vertical(direction, shading, blending, dithering,       \
4483  mask_evaluate, blend_mode)                                                    \
4484 do                                                                             \
4485 {                                                                              \
4486   error_step = delta_x * 2;                                                    \
4487   error_wrap = delta_y * 2;                                                    \
4488   error = delta_y;                                                             \
4489                                                                                \
4490   current_x = x_a;                                                             \
4491   set_line_gradients(y);                                                       \
4492                                                                                \
4493   for(current_y = y_a; compare_##direction(current_y, y_b);                    \
4494    update_##direction(current_y))                                              \
4495   {                                                                            \
4496     draw_pixel_line(current_x, current_y, shading, blending, dithering,        \
4497      mask_evaluate, blend_mode);                                               \
4498     error += error_step;                                                       \
4499     update_vram_row_##direction();                                             \
4500                                                                                \
4501     if(error > error_wrap)                                                     \
4502     {                                                                          \
4503       vram_ptr++;                                                              \
4504       current_x++;                                                             \
4505       error -= error_wrap;                                                     \
4506     }                                                                          \
4507   }                                                                            \
4508 } while(0)                                                                     \
4509
4510
4511 #define render_line_body(shading, blending, dithering, mask_evaluate,          \
4512  blend_mode)                                                                   \
4513   if(delta_y < 0)                                                              \
4514   {                                                                            \
4515     delta_y *= -1;                                                             \
4516                                                                                \
4517     if(delta_x > delta_y)                                                      \
4518     {                                                                          \
4519       draw_line_span_horizontal(decrement, shading, blending, dithering,       \
4520        mask_evaluate, blend_mode);                                             \
4521     }                                                                          \
4522     else                                                                       \
4523     {                                                                          \
4524       draw_line_span_vertical(decrement, shading, blending, dithering,         \
4525        mask_evaluate, blend_mode);                                             \
4526     }                                                                          \
4527   }                                                                            \
4528   else                                                                         \
4529   {                                                                            \
4530     if(delta_x > delta_y)                                                      \
4531     {                                                                          \
4532       draw_line_span_horizontal(increment, shading, blending, dithering,       \
4533        mask_evaluate, blend_mode);                                             \
4534     }                                                                          \
4535     else                                                                       \
4536     {                                                                          \
4537       draw_line_span_vertical(increment, shading, blending, dithering,         \
4538        mask_evaluate, blend_mode);                                             \
4539     }                                                                          \
4540   }                                                                            \
4541
4542
4543 void render_line(psx_gpu_struct *psx_gpu, vertex_struct *vertexes, u32 flags,
4544  u32 color, int double_resolution)
4545 {
4546   s32 color_r, color_g, color_b;
4547   u32 triangle_winding = 0;
4548
4549   fixed_type gradient_r = 0;
4550   fixed_type gradient_g = 0;
4551   fixed_type gradient_b = 0;
4552   fixed_type current_r = 0;
4553   fixed_type current_g = 0;
4554   fixed_type current_b = 0;
4555
4556   s32 y_a, y_b;
4557   s32 x_a, x_b;
4558
4559   s32 delta_x, delta_y;
4560
4561   s32 current_x;
4562   s32 current_y;
4563
4564   u32 error_step;
4565   u32 error;
4566   u32 error_wrap;
4567
4568   u16 *vram_ptr;
4569
4570   flush_render_block_buffer(psx_gpu);
4571   psx_gpu->primitive_type = PRIMITIVE_TYPE_LINE;
4572
4573   vertex_struct *vertex_a = &(vertexes[0]);
4574   vertex_struct *vertex_b = &(vertexes[1]);
4575
4576   u32 control_mask;
4577
4578 #ifdef PROFILE
4579   lines++;
4580 #endif
4581
4582   if(vertex_a->x >= vertex_b->x)
4583   {
4584     vertex_swap(vertex_a, vertex_b);
4585   }
4586
4587   x_a = vertex_a->x;
4588   x_b = vertex_b->x;
4589
4590   y_a = vertex_a->y;
4591   y_b = vertex_b->y;
4592
4593   delta_x = x_b - x_a;
4594   delta_y = y_b - y_a;
4595
4596   if(delta_x >= 1024 || delta_y >= 512 || delta_y <= -512)
4597     return;
4598
4599   if(double_resolution)
4600   {
4601     x_a *= 2;
4602     x_b *= 2;
4603     y_a *= 2;
4604     y_b *= 2;
4605     delta_x *= 2;
4606     delta_y *= 2;
4607   }
4608
4609   flags &= ~RENDER_FLAGS_TEXTURE_MAP;
4610
4611   vram_ptr = psx_gpu->vram_out_ptr + (y_a * 1024) + x_a;
4612
4613   control_mask = 0x0;
4614
4615   if(flags & RENDER_FLAGS_SHADE)
4616     control_mask |= 0x1;
4617
4618   if(flags & RENDER_FLAGS_BLEND)
4619   {
4620     control_mask |= 0x2;
4621     control_mask |= ((psx_gpu->render_state_base >> 6) & 0x3) << 4;
4622   }
4623
4624   if(psx_gpu->render_state_base & RENDER_STATE_DITHER)
4625     control_mask |= 0x4;
4626
4627   if(psx_gpu->render_state_base & RENDER_STATE_MASK_EVALUATE)
4628     control_mask |= 0x8;
4629
4630   switch(control_mask)
4631   {
4632     case 0x0:
4633       render_line_body(unshaded, unblended, undithered, no, none);
4634       break;
4635
4636     case 0x1:
4637       render_line_body(shaded, unblended, undithered, no, none);
4638       break;
4639
4640     case 0x2:
4641       render_line_body(unshaded, blended, undithered, no, average);
4642       break;
4643
4644     case 0x3:
4645       render_line_body(shaded, blended, undithered, no, average);
4646       break;
4647
4648     case 0x4:
4649       render_line_body(unshaded, unblended, dithered, no, none);
4650       break;
4651
4652     case 0x5:
4653       render_line_body(shaded, unblended, dithered, no, none);
4654       break;
4655
4656     case 0x6:
4657       render_line_body(unshaded, blended, dithered, no, average);
4658       break;
4659
4660     case 0x7:
4661       render_line_body(shaded, blended, dithered, no, average);
4662       break;
4663
4664     case 0x8:
4665       render_line_body(unshaded, unblended, undithered, yes, none);
4666       break;
4667
4668     case 0x9:
4669       render_line_body(shaded, unblended, undithered, yes, none);
4670       break;
4671
4672     case 0xA:
4673       render_line_body(unshaded, blended, undithered, yes, average);
4674       break;
4675
4676     case 0xB:
4677       render_line_body(shaded, blended, undithered, yes, average);
4678       break;
4679
4680     case 0xC:
4681       render_line_body(unshaded, unblended, dithered, yes, none);
4682       break;
4683
4684     case 0xD:
4685       render_line_body(shaded, unblended, dithered, yes, none);
4686       break;
4687
4688     case 0xE:
4689       render_line_body(unshaded, blended, dithered, yes, average);
4690       break;
4691
4692     case 0xF:
4693       render_line_body(shaded, blended, dithered, yes, average);
4694       break;
4695
4696     case 0x12:
4697       render_line_body(unshaded, blended, undithered, no, add);
4698       break;
4699
4700     case 0x13:
4701       render_line_body(shaded, blended, undithered, no, add);
4702       break;
4703
4704     case 0x16:
4705       render_line_body(unshaded, blended, dithered, no, add);
4706       break;
4707
4708     case 0x17:
4709       render_line_body(shaded, blended, dithered, no, add);
4710       break;
4711
4712     case 0x1A:
4713       render_line_body(unshaded, blended, undithered, yes, add);
4714       break;
4715
4716     case 0x1B:
4717       render_line_body(shaded, blended, undithered, yes, add);
4718       break;
4719
4720     case 0x1E:
4721       render_line_body(unshaded, blended, dithered, yes, add);
4722       break;
4723
4724     case 0x1F:
4725       render_line_body(shaded, blended, dithered, yes, add);
4726       break;
4727
4728     case 0x22:
4729       render_line_body(unshaded, blended, undithered, no, subtract);
4730       break;
4731
4732     case 0x23:
4733       render_line_body(shaded, blended, undithered, no, subtract);
4734       break;
4735
4736     case 0x26:
4737       render_line_body(unshaded, blended, dithered, no, subtract);
4738       break;
4739
4740     case 0x27:
4741       render_line_body(shaded, blended, dithered, no, subtract);
4742       break;
4743
4744     case 0x2A:
4745       render_line_body(unshaded, blended, undithered, yes, subtract);
4746       break;
4747
4748     case 0x2B:
4749       render_line_body(shaded, blended, undithered, yes, subtract);
4750       break;
4751
4752     case 0x2E:
4753       render_line_body(unshaded, blended, dithered, yes, subtract);
4754       break;
4755
4756     case 0x2F:
4757       render_line_body(shaded, blended, dithered, yes, subtract);
4758       break;
4759
4760     case 0x32:
4761       render_line_body(unshaded, blended, undithered, no, add_fourth);
4762       break;
4763
4764     case 0x33:
4765       render_line_body(shaded, blended, undithered, no, add_fourth);
4766       break;
4767
4768     case 0x36:
4769       render_line_body(unshaded, blended, dithered, no, add_fourth);
4770       break;
4771
4772     case 0x37:
4773       render_line_body(shaded, blended, dithered, no, add_fourth);
4774       break;
4775
4776     case 0x3A:
4777       render_line_body(unshaded, blended, undithered, yes, add_fourth);
4778       break;
4779
4780     case 0x3B:
4781       render_line_body(shaded, blended, undithered, yes, add_fourth);
4782       break;
4783
4784     case 0x3E:
4785       render_line_body(unshaded, blended, dithered, yes, add_fourth);
4786       break;
4787
4788     case 0x3F:
4789       render_line_body(shaded, blended, dithered, yes, add_fourth);
4790       break;
4791   }
4792 }
4793
4794
4795 void render_block_fill(psx_gpu_struct *psx_gpu, u32 color, u32 x, u32 y,
4796  u32 width, u32 height)
4797 {
4798   if((width == 0) || (height == 0))
4799     return;
4800
4801   invalidate_texture_cache_region(psx_gpu, x, y, x + width - 1, y + height - 1);
4802
4803   u32 r = color & 0xFF;
4804   u32 g = (color >> 8) & 0xFF;
4805   u32 b = (color >> 16) & 0xFF;
4806   u32 color_16bpp = (r >> 3) | ((g >> 3) << 5) | ((b >> 3) << 10) |
4807    psx_gpu->mask_msb;
4808   u32 color_32bpp = color_16bpp | (color_16bpp << 16);
4809
4810   u32 *vram_ptr = (u32 *)(psx_gpu->vram_out_ptr + x + (y * 1024));
4811
4812   u32 pitch = 512 - (width / 2);
4813   u32 num_width;
4814
4815   if(psx_gpu->render_mode & RENDER_INTERLACE_ENABLED)
4816   {
4817     pitch += 512;
4818     height /= 2;
4819
4820     if(psx_gpu->render_mode & RENDER_INTERLACE_ODD)
4821       vram_ptr += 512;
4822   }
4823
4824   while(height)
4825   {
4826     num_width = width;
4827     while(num_width)
4828     {
4829       vram_ptr[0] = color_32bpp;
4830       vram_ptr[1] = color_32bpp;
4831       vram_ptr[2] = color_32bpp;
4832       vram_ptr[3] = color_32bpp;
4833       vram_ptr[4] = color_32bpp;
4834       vram_ptr[5] = color_32bpp;
4835       vram_ptr[6] = color_32bpp;
4836       vram_ptr[7] = color_32bpp;
4837
4838       vram_ptr += 8;
4839       num_width -= 16;
4840     }
4841
4842     vram_ptr += pitch;
4843     height--;
4844   }
4845 }
4846
4847 void render_block_fill_enh(psx_gpu_struct *psx_gpu, u32 color, u32 x, u32 y,
4848  u32 width, u32 height)
4849 {
4850   if((width == 0) || (height == 0))
4851     return;
4852
4853   if(width > 1024)
4854     width = 1024;
4855
4856   u32 r = color & 0xFF;
4857   u32 g = (color >> 8) & 0xFF;
4858   u32 b = (color >> 16) & 0xFF;
4859   u32 color_16bpp = (r >> 3) | ((g >> 3) << 5) | ((b >> 3) << 10) |
4860    psx_gpu->mask_msb;
4861   u32 color_32bpp = color_16bpp | (color_16bpp << 16);
4862
4863   u32 *vram_ptr = (u32 *)(psx_gpu->vram_out_ptr + x + (y * 1024));
4864
4865   u32 pitch = 1024 / 2 - (width / 2);
4866   u32 num_width;
4867
4868   while(height)
4869   {
4870     num_width = width;
4871     while(num_width)
4872     {
4873       vram_ptr[0] = color_32bpp;
4874       vram_ptr[1] = color_32bpp;
4875       vram_ptr[2] = color_32bpp;
4876       vram_ptr[3] = color_32bpp;
4877       vram_ptr[4] = color_32bpp;
4878       vram_ptr[5] = color_32bpp;
4879       vram_ptr[6] = color_32bpp;
4880       vram_ptr[7] = color_32bpp;
4881
4882       vram_ptr += 8;
4883       num_width -= 16;
4884     }
4885
4886     vram_ptr += pitch;
4887     height--;
4888   }
4889 }
4890
4891 void render_block_copy(psx_gpu_struct *psx_gpu, u16 *source, u32 x, u32 y,
4892  u32 width, u32 height, u32 pitch)
4893 {
4894   u16 *vram_ptr = psx_gpu->vram_ptr + x + (y * 1024);
4895   u32 draw_x, draw_y;
4896   u32 mask_msb = psx_gpu->mask_msb;
4897
4898   if((width == 0) || (height == 0))
4899     return;
4900
4901   flush_render_block_buffer(psx_gpu);
4902   invalidate_texture_cache_region(psx_gpu, x, y, x + width - 1, y + height - 1);
4903
4904   for(draw_y = 0; draw_y < height; draw_y++)
4905   {
4906     for(draw_x = 0; draw_x < width; draw_x++)
4907     {
4908       vram_ptr[draw_x] = source[draw_x] | mask_msb;
4909     }
4910
4911     source += pitch;
4912     vram_ptr += 1024;
4913   }
4914 }
4915
4916 void render_block_move(psx_gpu_struct *psx_gpu, u32 source_x, u32 source_y,
4917  u32 dest_x, u32 dest_y, u32 width, u32 height)
4918 {
4919   render_block_copy(psx_gpu, psx_gpu->vram_ptr + source_x + (source_y * 1024),
4920    dest_x, dest_y, width, height, 1024);
4921 }
4922
4923
4924 void initialize_reciprocal_table(void)
4925 {
4926   u32 height;
4927   u32 height_normalized;
4928   u32 height_reciprocal;
4929   s32 shift;
4930
4931   for(height = 1; height < sizeof(reciprocal_table)
4932        / sizeof(reciprocal_table[0]); height++)
4933   {
4934     shift = __builtin_clz(height);
4935     height_normalized = height << shift;
4936     height_reciprocal = ((1ULL << 51) + (height_normalized - 1)) /
4937      height_normalized;
4938
4939     shift = 32 - (51 - shift);
4940
4941     reciprocal_table[height] = (height_reciprocal << 10) | shift;
4942   }
4943 }
4944
4945
4946 #define dither_table_row(a, b, c, d)                                           \
4947  ((a & 0xFF) | ((b & 0xFF) << 8) | ((c & 0xFF) << 16) | ((d & 0xFF) << 24))    \
4948
4949 void initialize_psx_gpu(psx_gpu_struct *psx_gpu, u16 *vram)
4950 {
4951   vec_8x16u test_mask =
4952    { { { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 } } };
4953
4954   psx_gpu->test_mask = test_mask;
4955
4956   psx_gpu->dirty_textures_4bpp_mask = 0xFFFFFFFF;
4957   psx_gpu->dirty_textures_8bpp_mask = 0xFFFFFFFF;
4958   psx_gpu->dirty_textures_8bpp_alternate_mask = 0xFFFFFFFF;
4959   psx_gpu->viewport_mask = 0;
4960   psx_gpu->current_texture_page = 0;
4961   psx_gpu->current_texture_mask = 0;
4962   psx_gpu->last_8bpp_texture_page = 0;
4963
4964   psx_gpu->clut_settings = 0;
4965   psx_gpu->texture_settings = 0;
4966   psx_gpu->render_state = 0;
4967   psx_gpu->render_state_base = 0;
4968   psx_gpu->num_blocks = 0;
4969   psx_gpu->uvrgb_phase = 0x8000;
4970
4971   psx_gpu->vram_ptr = vram;
4972   psx_gpu->vram_out_ptr = vram;
4973
4974   psx_gpu->texture_page_base = psx_gpu->vram_ptr;
4975   psx_gpu->texture_page_ptr = psx_gpu->vram_ptr;
4976   psx_gpu->clut_ptr = psx_gpu->vram_ptr;
4977
4978   psx_gpu->mask_msb = 0;
4979
4980   psx_gpu->texture_window_x = 0;
4981   psx_gpu->texture_window_y = 0;
4982   psx_gpu->texture_mask_width = 0xFF;
4983   psx_gpu->texture_mask_height = 0xFF;
4984
4985   psx_gpu->render_mode = 0;
4986
4987   memset(psx_gpu->vram_ptr, 0, sizeof(u16) * 1024 * 512);
4988
4989   initialize_reciprocal_table();
4990   psx_gpu->reciprocal_table_ptr = reciprocal_table;
4991
4992   //    00 01 10 11
4993   // 00  0  4  1  5
4994   // 01  6  2  7  3
4995   // 10  1  5  0  4
4996   // 11  7  3  6  2
4997   // (minus ones(4) * 4)
4998
4999   // d0: (1 3 5 7): x1 ^ y1
5000   // d1: (2 3 6 7): y0
5001   // d2: (4 5 6 7): x0 ^ y0
5002
5003   psx_gpu->dither_table[0] = dither_table_row(-4, 0, -3, 1);
5004   psx_gpu->dither_table[1] = dither_table_row(2, -2, 3, -1);
5005   psx_gpu->dither_table[2] = dither_table_row(-3, 1, -4, 0);
5006   psx_gpu->dither_table[3] = dither_table_row(3, -1, 2, -2);
5007
5008   psx_gpu->primitive_type = PRIMITIVE_TYPE_UNKNOWN;
5009
5010   psx_gpu->saved_hres = 256;
5011 }
5012
5013 u64 get_us(void)
5014 {
5015   struct timeval tv;
5016   gettimeofday(&tv, NULL);
5017
5018   return (tv.tv_sec * 1000000ULL) + tv.tv_usec;
5019 }
5020
5021 #if 0 //def NEON_BUILD
5022
5023 u32 get_counter()
5024 {
5025   u32 counter;
5026   __asm__ volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(counter));
5027
5028   return counter;
5029 }
5030
5031 void init_counter(void)
5032 {
5033   u32 value;
5034   asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(value));
5035   value |= 5; // master enable, ccnt reset
5036   value &= ~8; // ccnt divider 0
5037   asm volatile("mcr p15, 0, %0, c9, c12, 0" :: "r"(value));
5038   // enable cycle counter
5039   asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(1 << 31));
5040 }
5041
5042 void triangle_benchmark(psx_gpu_struct *psx_gpu)
5043 {
5044   u32 i;
5045
5046   u32 ticks;
5047   u32 ticks_elapsed;
5048
5049   const u32 iterations = 500000;
5050
5051   psx_gpu->num_blocks = 64;
5052   psx_gpu->clut_ptr = psx_gpu->vram_ptr;
5053
5054   for(i = 0; i < 64; i++)
5055   {
5056     memset(&(psx_gpu->blocks[i].r), 0, 16);
5057   }
5058
5059   init_counter();
5060
5061   ticks = get_counter();
5062
5063   for(i = 0; i < iterations; i++)
5064   {
5065     texture_sprite_blocks_8bpp(psx_gpu);
5066   }
5067
5068   ticks_elapsed = get_counter() - ticks;
5069
5070   printf("benchmark: %lf cycles\n", (double)ticks_elapsed / (iterations * 64));
5071 }
5072
5073 #endif
5074
5075 #include "psx_gpu_4x.c"
5076
5077 // vim:ts=2:sw=2:expandtab