libpcsxcore/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2011 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdlib.h>
  22 #include <stdint.h> //include for uint64_t
  23 #include <assert.h>
  24 #include <errno.h>
  25 #include <sys/mman.h>
  26 #ifdef __MACH__
  27 #include <libkern/OSCacheControl.h>
  28 #endif
  29 #ifdef _3DS
  30 #include <3ds_utils.h>
  31 #endif
  32 #ifdef VITA
  33 #include <psp2/kernel/sysmem.h>
  34 static int sceBlock;
  35 #endif
  36
  37 #include "new_dynarec_config.h"
  38 #include "../psxhle.h"
  39 #include "../psxinterpreter.h"
  40 #include "../gte.h"
  41 #include "emu_if.h" // emulator interface
  42
  43 #define noinline __attribute__((noinline,noclone))
  44 #ifndef ARRAY_SIZE
  45 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
  46 #endif
  47
  48 //#define DISASM
  49 //#define assem_debug printf
  50 //#define inv_debug printf
  51 #define assem_debug(...)
  52 #define inv_debug(...)
  53
  54 #ifdef __i386__
  55 #include "assem_x86.h"
  56 #endif
  57 #ifdef __x86_64__
  58 #include "assem_x64.h"
  59 #endif
  60 #ifdef __arm__
  61 #include "assem_arm.h"
  62 #endif
  63 #ifdef __aarch64__
  64 #include "assem_arm64.h"
  65 #endif
  66
  67 #define RAM_SIZE 0x200000
  68 #define MAXBLOCK 4096
  69 #define MAX_OUTPUT_BLOCK_SIZE 262144
  70
  71 struct ndrc_mem
  72 {
  73   u_char translation_cache[1 << TARGET_SIZE_2];
  74   struct
  75   {
  76     struct tramp_insns ops[2048 / sizeof(struct tramp_insns)];
  77     const void *f[2048 / sizeof(void *)];
  78   } tramp;
  79 };
  80
  81 #ifdef BASE_ADDR_DYNAMIC
  82 static struct ndrc_mem *ndrc;
  83 #else
  84 static struct ndrc_mem ndrc_ __attribute__((aligned(4096)));
  85 static struct ndrc_mem *ndrc = &ndrc_;
  86 #endif
  87
  88 // stubs
  89 enum stub_type {
  90   CC_STUB = 1,
  91   FP_STUB = 2,
  92   LOADB_STUB = 3,
  93   LOADH_STUB = 4,
  94   LOADW_STUB = 5,
  95   LOADD_STUB = 6,
  96   LOADBU_STUB = 7,
  97   LOADHU_STUB = 8,
  98   STOREB_STUB = 9,
  99   STOREH_STUB = 10,
 100   STOREW_STUB = 11,
 101   STORED_STUB = 12,
 102   STORELR_STUB = 13,
 103   INVCODE_STUB = 14,
 104 };
 105
 106 struct regstat
 107 {
 108   signed char regmap_entry[HOST_REGS];
 109   signed char regmap[HOST_REGS];
 110   uint64_t wasdirty;
 111   uint64_t dirty;
 112   uint64_t u;
 113   u_int wasconst;
 114   u_int isconst;
 115   u_int loadedconst;             // host regs that have constants loaded
 116   u_int waswritten;              // MIPS regs that were used as store base before
 117 };
 118
 119 // note: asm depends on this layout
 120 struct ll_entry
 121 {
 122   u_int vaddr;
 123   u_int reg_sv_flags;
 124   void *addr;
 125   struct ll_entry *next;
 126 };
 127
 128 struct ht_entry
 129 {
 130   u_int vaddr[2];
 131   void *tcaddr[2];
 132 };
 133
 134 struct code_stub
 135 {
 136   enum stub_type type;
 137   void *addr;
 138   void *retaddr;
 139   u_int a;
 140   uintptr_t b;
 141   uintptr_t c;
 142   u_int d;
 143   u_int e;
 144 };
 145
 146 struct link_entry
 147 {
 148   void *addr;
 149   u_int target;
 150   u_int ext;
 151 };
 152
 153   // used by asm:
 154   u_char *out;
 155   struct ht_entry hash_table[65536]  __attribute__((aligned(16)));
 156   struct ll_entry *jump_in[4096] __attribute__((aligned(16)));
 157   struct ll_entry *jump_dirty[4096];
 158
 159   static struct ll_entry *jump_out[4096];
 160   static u_int start;
 161   static u_int *source;
 162   static char insn[MAXBLOCK][10];
 163   static u_char itype[MAXBLOCK];
 164   static u_char opcode[MAXBLOCK];
 165   static u_char opcode2[MAXBLOCK];
 166   static u_char bt[MAXBLOCK];
 167   static u_char rs1[MAXBLOCK];
 168   static u_char rs2[MAXBLOCK];
 169   static u_char rt1[MAXBLOCK];
 170   static u_char rt2[MAXBLOCK];
 171   static u_char dep1[MAXBLOCK];
 172   static u_char dep2[MAXBLOCK];
 173   static u_char lt1[MAXBLOCK];
 174   static uint64_t gte_rs[MAXBLOCK]; // gte: 32 data and 32 ctl regs
 175   static uint64_t gte_rt[MAXBLOCK];
 176   static uint64_t gte_unneeded[MAXBLOCK];
 177   static u_int smrv[32]; // speculated MIPS register values
 178   static u_int smrv_strong; // mask or regs that are likely to have correct values
 179   static u_int smrv_weak; // same, but somewhat less likely
 180   static u_int smrv_strong_next; // same, but after current insn executes
 181   static u_int smrv_weak_next;
 182   static int imm[MAXBLOCK];
 183   static u_int ba[MAXBLOCK];
 184   static char likely[MAXBLOCK];
 185   static char is_ds[MAXBLOCK];
 186   static char ooo[MAXBLOCK];
 187   static uint64_t unneeded_reg[MAXBLOCK];
 188   static uint64_t branch_unneeded_reg[MAXBLOCK];
 189   static signed char regmap_pre[MAXBLOCK][HOST_REGS]; // pre-instruction i?
 190   // contains 'real' consts at [i] insn, but may differ from what's actually
 191   // loaded in host reg as 'final' value is always loaded, see get_final_value()
 192   static uint32_t current_constmap[HOST_REGS];
 193   static uint32_t constmap[MAXBLOCK][HOST_REGS];
 194   static struct regstat regs[MAXBLOCK];
 195   static struct regstat branch_regs[MAXBLOCK];
 196   static signed char minimum_free_regs[MAXBLOCK];
 197   static u_int needed_reg[MAXBLOCK];
 198   static u_int wont_dirty[MAXBLOCK];
 199   static u_int will_dirty[MAXBLOCK];
 200   static int ccadj[MAXBLOCK];
 201   static int slen;
 202   static void *instr_addr[MAXBLOCK];
 203   static struct link_entry link_addr[MAXBLOCK];
 204   static int linkcount;
 205   static struct code_stub stubs[MAXBLOCK*3];
 206   static int stubcount;
 207   static u_int literals[1024][2];
 208   static int literalcount;
 209   static int is_delayslot;
 210   static char shadow[1048576]  __attribute__((aligned(16)));
 211   static void *copy;
 212   static int expirep;
 213   static u_int stop_after_jal;
 214 #ifndef RAM_FIXED
 215   static uintptr_t ram_offset;
 216 #else
 217   static const uintptr_t ram_offset=0;
 218 #endif
 219
 220   int new_dynarec_hacks;
 221   int new_dynarec_hacks_pergame;
 222   int new_dynarec_did_compile;
 223
 224   #define HACK_ENABLED(x) ((new_dynarec_hacks | new_dynarec_hacks_pergame) & (x))
 225
 226   extern int cycle_count; // ... until end of the timeslice, counts -N -> 0
 227   extern int last_count;  // last absolute target, often = next_interupt
 228   extern int pcaddr;
 229   extern int pending_exception;
 230   extern int branch_target;
 231   extern uintptr_t mini_ht[32][2];
 232   extern u_char restore_candidate[512];
 233
 234   /* registers that may be allocated */
 235   /* 1-31 gpr */
 236 #define LOREG 32 // lo
 237 #define HIREG 33 // hi
 238 //#define FSREG 34 // FPU status (FCSR)
 239 #define CSREG 35 // Coprocessor status
 240 #define CCREG 36 // Cycle count
 241 #define INVCP 37 // Pointer to invalid_code
 242 //#define MMREG 38 // Pointer to memory_map
 243 //#define ROREG 39 // ram offset (if rdram!=0x80000000)
 244 #define TEMPREG 40
 245 #define FTEMP 40 // FPU temporary register
 246 #define PTEMP 41 // Prefetch temporary register
 247 //#define TLREG 42 // TLB mapping offset
 248 #define RHASH 43 // Return address hash
 249 #define RHTBL 44 // Return address hash table address
 250 #define RTEMP 45 // JR/JALR address register
 251 #define MAXREG 45
 252 #define AGEN1 46 // Address generation temporary register
 253 //#define AGEN2 47 // Address generation temporary register
 254 //#define MGEN1 48 // Maptable address generation temporary register
 255 //#define MGEN2 49 // Maptable address generation temporary register
 256 #define BTREG 50 // Branch target temporary register
 257
 258   /* instruction types */
 259 #define NOP 0     // No operation
 260 #define LOAD 1    // Load
 261 #define STORE 2   // Store
 262 #define LOADLR 3  // Unaligned load
 263 #define STORELR 4 // Unaligned store
 264 #define MOV 5     // Move
 265 #define ALU 6     // Arithmetic/logic
 266 #define MULTDIV 7 // Multiply/divide
 267 #define SHIFT 8   // Shift by register
 268 #define SHIFTIMM 9// Shift by immediate
 269 #define IMM16 10  // 16-bit immediate
 270 #define RJUMP 11  // Unconditional jump to register
 271 #define UJUMP 12  // Unconditional jump
 272 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 273 #define SJUMP 14  // Conditional branch (regimm format)
 274 #define COP0 15   // Coprocessor 0
 275 #define COP1 16   // Coprocessor 1
 276 #define C1LS 17   // Coprocessor 1 load/store
 277 //#define FJUMP 18  // Conditional branch (floating point)
 278 //#define FLOAT 19  // Floating point unit
 279 //#define FCONV 20  // Convert integer to float
 280 //#define FCOMP 21  // Floating point compare (sets FSREG)
 281 #define SYSCALL 22// SYSCALL
 282 #define OTHER 23  // Other
 283 #define SPAN 24   // Branch/delay slot spans 2 pages
 284 #define NI 25     // Not implemented
 285 #define HLECALL 26// PCSX fake opcodes for HLE
 286 #define COP2 27   // Coprocessor 2 move
 287 #define C2LS 28   // Coprocessor 2 load/store
 288 #define C2OP 29   // Coprocessor 2 operation
 289 #define INTCALL 30// Call interpreter to handle rare corner cases
 290
 291   /* branch codes */
 292 #define TAKEN 1
 293 #define NOTTAKEN 2
 294 #define NULLDS 3
 295
 296 #define DJT_1 (void *)1l // no function, just a label in assem_debug log
 297 #define DJT_2 (void *)2l
 298
 299 // asm linkage
 300 int new_recompile_block(u_int addr);
 301 void *get_addr_ht(u_int vaddr);
 302 void invalidate_block(u_int block);
 303 void invalidate_addr(u_int addr);
 304 void remove_hash(int vaddr);
 305 void dyna_linker();
 306 void dyna_linker_ds();
 307 void verify_code();
 308 void verify_code_ds();
 309 void cc_interrupt();
 310 void fp_exception();
 311 void fp_exception_ds();
 312 void jump_to_new_pc();
 313 void call_gteStall();
 314 void new_dyna_leave();
 315
 316 // Needed by assembler
 317 static void wb_register(signed char r,signed char regmap[],uint64_t dirty);
 318 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty);
 319 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr);
 320 static void load_all_regs(signed char i_regmap[]);
 321 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 322 static void load_regs_entry(int t);
 323 static void load_all_consts(signed char regmap[],u_int dirty,int i);
 324 static u_int get_host_reglist(const signed char *regmap);
 325
 326 static int verify_dirty(const u_int *ptr);
 327 static int get_final_value(int hr, int i, int *value);
 328 static void add_stub(enum stub_type type, void *addr, void *retaddr,
 329   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e);
 330 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
 331   int i, int addr_reg, const struct regstat *i_regs, int ccadj, u_int reglist);
 332 static void add_to_linker(void *addr, u_int target, int ext);
 333 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override);
 334 static void *get_direct_memhandler(void *table, u_int addr,
 335   enum stub_type type, uintptr_t *addr_host);
 336 static void cop2_call_stall_check(u_int op, int i, const struct regstat *i_regs, u_int reglist);
 337 static void pass_args(int a0, int a1);
 338 static void emit_far_jump(const void *f);
 339 static void emit_far_call(const void *f);
 340
 341 static void mprotect_w_x(void *start, void *end, int is_x)
 342 {
 343 #ifdef NO_WRITE_EXEC
 344   #if defined(VITA)
 345   // *Open* enables write on all memory that was
 346   // allocated by sceKernelAllocMemBlockForVM()?
 347   if (is_x)
 348     sceKernelCloseVMDomain();
 349   else
 350     sceKernelOpenVMDomain();
 351   #else
 352   u_long mstart = (u_long)start & ~4095ul;
 353   u_long mend = (u_long)end;
 354   if (mprotect((void *)mstart, mend - mstart,
 355                PROT_READ | (is_x ? PROT_EXEC : PROT_WRITE)) != 0)
 356     SysPrintf("mprotect(%c) failed: %s\n", is_x ? 'x' : 'w', strerror(errno));
 357   #endif
 358 #endif
 359 }
 360
 361 static void start_tcache_write(void *start, void *end)
 362 {
 363   mprotect_w_x(start, end, 0);
 364 }
 365
 366 static void end_tcache_write(void *start, void *end)
 367 {
 368 #if defined(__arm__) || defined(__aarch64__)
 369   size_t len = (char *)end - (char *)start;
 370   #if   defined(__BLACKBERRY_QNX__)
 371   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);
 372   #elif defined(__MACH__)
 373   sys_cache_control(kCacheFunctionPrepareForExecution, start, len);
 374   #elif defined(VITA)
 375   sceKernelSyncVMDomain(sceBlock, start, len);
 376   #elif defined(_3DS)
 377   ctr_flush_invalidate_cache();
 378   #elif defined(__aarch64__)
 379   // as of 2021, __clear_cache() is still broken on arm64
 380   // so here is a custom one :(
 381   clear_cache_arm64(start, end);
 382   #else
 383   __clear_cache(start, end);
 384   #endif
 385   (void)len;
 386 #endif
 387
 388   mprotect_w_x(start, end, 1);
 389 }
 390
 391 static void *start_block(void)
 392 {
 393   u_char *end = out + MAX_OUTPUT_BLOCK_SIZE;
 394   if (end > ndrc->translation_cache + sizeof(ndrc->translation_cache))
 395     end = ndrc->translation_cache + sizeof(ndrc->translation_cache);
 396   start_tcache_write(out, end);
 397   return out;
 398 }
 399
 400 static void end_block(void *start)
 401 {
 402   end_tcache_write(start, out);
 403 }
 404
 405 // also takes care of w^x mappings when patching code
 406 static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)];
 407
 408 static void mark_clear_cache(void *target)
 409 {
 410   uintptr_t offset = (u_char *)target - ndrc->translation_cache;
 411   u_int mask = 1u << ((offset >> 12) & 31);
 412   if (!(needs_clear_cache[offset >> 17] & mask)) {
 413     char *start = (char *)((uintptr_t)target & ~4095l);
 414     start_tcache_write(start, start + 4095);
 415     needs_clear_cache[offset >> 17] |= mask;
 416   }
 417 }
 418
 419 // Clearing the cache is rather slow on ARM Linux, so mark the areas
 420 // that need to be cleared, and then only clear these areas once.
 421 static void do_clear_cache(void)
 422 {
 423   int i, j;
 424   for (i = 0; i < (1<<(TARGET_SIZE_2-17)); i++)
 425   {
 426     u_int bitmap = needs_clear_cache[i];
 427     if (!bitmap)
 428       continue;
 429     for (j = 0; j < 32; j++)
 430     {
 431       u_char *start, *end;
 432       if (!(bitmap & (1<<j)))
 433         continue;
 434
 435       start = ndrc->translation_cache + i*131072 + j*4096;
 436       end = start + 4095;
 437       for (j++; j < 32; j++) {
 438         if (!(bitmap & (1<<j)))
 439           break;
 440         end += 4096;
 441       }
 442       end_tcache_write(start, end);
 443     }
 444     needs_clear_cache[i] = 0;
 445   }
 446 }
 447
 448 //#define DEBUG_CYCLE_COUNT 1
 449
 450 #define NO_CYCLE_PENALTY_THR 12
 451
 452 int cycle_multiplier; // 100 for 1.0
 453 int cycle_multiplier_override;
 454
 455 static int CLOCK_ADJUST(int x)
 456 {
 457   int m = cycle_multiplier_override
 458         ? cycle_multiplier_override : cycle_multiplier;
 459   int s=(x>>31)|1;
 460   return (x * m + s * 50) / 100;
 461 }
 462
 463 // is the op an unconditional jump?
 464 static int is_ujump(int i)
 465 {
 466   return itype[i] == UJUMP || itype[i] == RJUMP
 467     || (source[i] >> 16) == 0x1000; // beq r0, r0, offset // b offset
 468 }
 469
 470 static int is_jump(int i)
 471 {
 472   return itype[i] == RJUMP || itype[i] == UJUMP || itype[i] == CJUMP || itype[i] == SJUMP;
 473 }
 474
 475 static u_int get_page(u_int vaddr)
 476 {
 477   u_int page=vaddr&~0xe0000000;
 478   if (page < 0x1000000)
 479     page &= ~0x0e00000; // RAM mirrors
 480   page>>=12;
 481   if(page>2048) page=2048+(page&2047);
 482   return page;
 483 }
 484
 485 // no virtual mem in PCSX
 486 static u_int get_vpage(u_int vaddr)
 487 {
 488   return get_page(vaddr);
 489 }
 490
 491 static struct ht_entry *hash_table_get(u_int vaddr)
 492 {
 493   return &hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 494 }
 495
 496 static void hash_table_add(struct ht_entry *ht_bin, u_int vaddr, void *tcaddr)
 497 {
 498   ht_bin->vaddr[1] = ht_bin->vaddr[0];
 499   ht_bin->tcaddr[1] = ht_bin->tcaddr[0];
 500   ht_bin->vaddr[0] = vaddr;
 501   ht_bin->tcaddr[0] = tcaddr;
 502 }
 503
 504 // some messy ari64's code, seems to rely on unsigned 32bit overflow
 505 static int doesnt_expire_soon(void *tcaddr)
 506 {
 507   u_int diff = (u_int)((u_char *)tcaddr - out) << (32-TARGET_SIZE_2);
 508   return diff > (u_int)(0x60000000 + (MAX_OUTPUT_BLOCK_SIZE << (32-TARGET_SIZE_2)));
 509 }
 510
 511 // Get address from virtual address
 512 // This is called from the recompiled JR/JALR instructions
 513 void noinline *get_addr(u_int vaddr)
 514 {
 515   u_int page=get_page(vaddr);
 516   u_int vpage=get_vpage(vaddr);
 517   struct ll_entry *head;
 518   //printf("TRACE: count=%d next=%d (get_addr %x,page %d)\n",Count,next_interupt,vaddr,page);
 519   head=jump_in[page];
 520   while(head!=NULL) {
 521     if(head->vaddr==vaddr) {
 522   //printf("TRACE: count=%d next=%d (get_addr match %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 523       hash_table_add(hash_table_get(vaddr), vaddr, head->addr);
 524       return head->addr;
 525     }
 526     head=head->next;
 527   }
 528   head=jump_dirty[vpage];
 529   while(head!=NULL) {
 530     if(head->vaddr==vaddr) {
 531       //printf("TRACE: count=%d next=%d (get_addr match dirty %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 532       // Don't restore blocks which are about to expire from the cache
 533       if (doesnt_expire_soon(head->addr))
 534       if (verify_dirty(head->addr)) {
 535         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 536         invalid_code[vaddr>>12]=0;
 537         inv_code_start=inv_code_end=~0;
 538         if(vpage<2048) {
 539           restore_candidate[vpage>>3]|=1<<(vpage&7);
 540         }
 541         else restore_candidate[page>>3]|=1<<(page&7);
 542         struct ht_entry *ht_bin = hash_table_get(vaddr);
 543         if (ht_bin->vaddr[0] == vaddr)
 544           ht_bin->tcaddr[0] = head->addr; // Replace existing entry
 545         else
 546           hash_table_add(ht_bin, vaddr, head->addr);
 547
 548         return head->addr;
 549       }
 550     }
 551     head=head->next;
 552   }
 553   //printf("TRACE: count=%d next=%d (get_addr no-match %x)\n",Count,next_interupt,vaddr);
 554   int r=new_recompile_block(vaddr);
 555   if(r==0) return get_addr(vaddr);
 556   // Execute in unmapped page, generate pagefault execption
 557   Status|=2;
 558   Cause=(vaddr<<31)|0x8;
 559   EPC=(vaddr&1)?vaddr-5:vaddr;
 560   BadVAddr=(vaddr&~1);
 561   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 562   EntryHi=BadVAddr&0xFFFFE000;
 563   return get_addr_ht(0x80000000);
 564 }
 565 // Look up address in hash table first
 566 void *get_addr_ht(u_int vaddr)
 567 {
 568   //printf("TRACE: count=%d next=%d (get_addr_ht %x)\n",Count,next_interupt,vaddr);
 569   const struct ht_entry *ht_bin = hash_table_get(vaddr);
 570   if (ht_bin->vaddr[0] == vaddr) return ht_bin->tcaddr[0];
 571   if (ht_bin->vaddr[1] == vaddr) return ht_bin->tcaddr[1];
 572   return get_addr(vaddr);
 573 }
 574
 575 void clear_all_regs(signed char regmap[])
 576 {
 577   int hr;
 578   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 579 }
 580
 581 static signed char get_reg(const signed char regmap[],int r)
 582 {
 583   int hr;
 584   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 585   return -1;
 586 }
 587
 588 // Find a register that is available for two consecutive cycles
 589 static signed char get_reg2(signed char regmap1[], const signed char regmap2[], int r)
 590 {
 591   int hr;
 592   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 593   return -1;
 594 }
 595
 596 int count_free_regs(signed char regmap[])
 597 {
 598   int count=0;
 599   int hr;
 600   for(hr=0;hr<HOST_REGS;hr++)
 601   {
 602     if(hr!=EXCLUDE_REG) {
 603       if(regmap[hr]<0) count++;
 604     }
 605   }
 606   return count;
 607 }
 608
 609 void dirty_reg(struct regstat *cur,signed char reg)
 610 {
 611   int hr;
 612   if(!reg) return;
 613   for (hr=0;hr<HOST_REGS;hr++) {
 614     if((cur->regmap[hr]&63)==reg) {
 615       cur->dirty|=1<<hr;
 616     }
 617   }
 618 }
 619
 620 static void set_const(struct regstat *cur, signed char reg, uint32_t value)
 621 {
 622   int hr;
 623   if(!reg) return;
 624   for (hr=0;hr<HOST_REGS;hr++) {
 625     if(cur->regmap[hr]==reg) {
 626       cur->isconst|=1<<hr;
 627       current_constmap[hr]=value;
 628     }
 629   }
 630 }
 631
 632 static void clear_const(struct regstat *cur, signed char reg)
 633 {
 634   int hr;
 635   if(!reg) return;
 636   for (hr=0;hr<HOST_REGS;hr++) {
 637     if((cur->regmap[hr]&63)==reg) {
 638       cur->isconst&=~(1<<hr);
 639     }
 640   }
 641 }
 642
 643 static int is_const(struct regstat *cur, signed char reg)
 644 {
 645   int hr;
 646   if(reg<0) return 0;
 647   if(!reg) return 1;
 648   for (hr=0;hr<HOST_REGS;hr++) {
 649     if((cur->regmap[hr]&63)==reg) {
 650       return (cur->isconst>>hr)&1;
 651     }
 652   }
 653   return 0;
 654 }
 655
 656 static uint32_t get_const(struct regstat *cur, signed char reg)
 657 {
 658   int hr;
 659   if(!reg) return 0;
 660   for (hr=0;hr<HOST_REGS;hr++) {
 661     if(cur->regmap[hr]==reg) {
 662       return current_constmap[hr];
 663     }
 664   }
 665   SysPrintf("Unknown constant in r%d\n",reg);
 666   abort();
 667 }
 668
 669 // Least soon needed registers
 670 // Look at the next ten instructions and see which registers
 671 // will be used.  Try not to reallocate these.
 672 void lsn(u_char hsn[], int i, int *preferred_reg)
 673 {
 674   int j;
 675   int b=-1;
 676   for(j=0;j<9;j++)
 677   {
 678     if(i+j>=slen) {
 679       j=slen-i-1;
 680       break;
 681     }
 682     if (is_ujump(i+j))
 683     {
 684       // Don't go past an unconditonal jump
 685       j++;
 686       break;
 687     }
 688   }
 689   for(;j>=0;j--)
 690   {
 691     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 692     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 693     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 694     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 695     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 696       // Stores can allocate zero
 697       hsn[rs1[i+j]]=j;
 698       hsn[rs2[i+j]]=j;
 699     }
 700     // On some architectures stores need invc_ptr
 701     #if defined(HOST_IMM8)
 702     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39 || (opcode[i+j]&0x3b)==0x3a) {
 703       hsn[INVCP]=j;
 704     }
 705     #endif
 706     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 707     {
 708       hsn[CCREG]=j;
 709       b=j;
 710     }
 711   }
 712   if(b>=0)
 713   {
 714     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 715     {
 716       // Follow first branch
 717       int t=(ba[i+b]-start)>>2;
 718       j=7-b;if(t+j>=slen) j=slen-t-1;
 719       for(;j>=0;j--)
 720       {
 721         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 722         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 723         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 724         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 725       }
 726     }
 727     // TODO: preferred register based on backward branch
 728   }
 729   // Delay slot should preferably not overwrite branch conditions or cycle count
 730   if (i > 0 && is_jump(i-1)) {
 731     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 732     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 733     hsn[CCREG]=1;
 734     // ...or hash tables
 735     hsn[RHASH]=1;
 736     hsn[RHTBL]=1;
 737   }
 738   // Coprocessor load/store needs FTEMP, even if not declared
 739   if(itype[i]==C1LS||itype[i]==C2LS) {
 740     hsn[FTEMP]=0;
 741   }
 742   // Load L/R also uses FTEMP as a temporary register
 743   if(itype[i]==LOADLR) {
 744     hsn[FTEMP]=0;
 745   }
 746   // Also SWL/SWR/SDL/SDR
 747   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) {
 748     hsn[FTEMP]=0;
 749   }
 750   // Don't remove the miniht registers
 751   if(itype[i]==UJUMP||itype[i]==RJUMP)
 752   {
 753     hsn[RHASH]=0;
 754     hsn[RHTBL]=0;
 755   }
 756 }
 757
 758 // We only want to allocate registers if we're going to use them again soon
 759 int needed_again(int r, int i)
 760 {
 761   int j;
 762   int b=-1;
 763   int rn=10;
 764
 765   if (i > 0 && is_ujump(i-1))
 766   {
 767     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 768       return 0; // Don't need any registers if exiting the block
 769   }
 770   for(j=0;j<9;j++)
 771   {
 772     if(i+j>=slen) {
 773       j=slen-i-1;
 774       break;
 775     }
 776     if (is_ujump(i+j))
 777     {
 778       // Don't go past an unconditonal jump
 779       j++;
 780       break;
 781     }
 782     if(itype[i+j]==SYSCALL||itype[i+j]==HLECALL||itype[i+j]==INTCALL||((source[i+j]&0xfc00003f)==0x0d))
 783     {
 784       break;
 785     }
 786   }
 787   for(;j>=1;j--)
 788   {
 789     if(rs1[i+j]==r) rn=j;
 790     if(rs2[i+j]==r) rn=j;
 791     if((unneeded_reg[i+j]>>r)&1) rn=10;
 792     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 793     {
 794       b=j;
 795     }
 796   }
 797   /*
 798   if(b>=0)
 799   {
 800     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 801     {
 802       // Follow first branch
 803       int o=rn;
 804       int t=(ba[i+b]-start)>>2;
 805       j=7-b;if(t+j>=slen) j=slen-t-1;
 806       for(;j>=0;j--)
 807       {
 808         if(!((unneeded_reg[t+j]>>r)&1)) {
 809           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 810           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 811         }
 812         else rn=o;
 813       }
 814     }
 815   }*/
 816   if(rn<10) return 1;
 817   (void)b;
 818   return 0;
 819 }
 820
 821 // Try to match register allocations at the end of a loop with those
 822 // at the beginning
 823 int loop_reg(int i, int r, int hr)
 824 {
 825   int j,k;
 826   for(j=0;j<9;j++)
 827   {
 828     if(i+j>=slen) {
 829       j=slen-i-1;
 830       break;
 831     }
 832     if (is_ujump(i+j))
 833     {
 834       // Don't go past an unconditonal jump
 835       j++;
 836       break;
 837     }
 838   }
 839   k=0;
 840   if(i>0){
 841     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)
 842       k--;
 843   }
 844   for(;k<j;k++)
 845   {
 846     assert(r < 64);
 847     if((unneeded_reg[i+k]>>r)&1) return hr;
 848     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP))
 849     {
 850       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 851       {
 852         int t=(ba[i+k]-start)>>2;
 853         int reg=get_reg(regs[t].regmap_entry,r);
 854         if(reg>=0) return reg;
 855         //reg=get_reg(regs[t+1].regmap_entry,r);
 856         //if(reg>=0) return reg;
 857       }
 858     }
 859   }
 860   return hr;
 861 }
 862
 863
 864 // Allocate every register, preserving source/target regs
 865 void alloc_all(struct regstat *cur,int i)
 866 {
 867   int hr;
 868
 869   for(hr=0;hr<HOST_REGS;hr++) {
 870     if(hr!=EXCLUDE_REG) {
 871       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 872          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 873       {
 874         cur->regmap[hr]=-1;
 875         cur->dirty&=~(1<<hr);
 876       }
 877       // Don't need zeros
 878       if((cur->regmap[hr]&63)==0)
 879       {
 880         cur->regmap[hr]=-1;
 881         cur->dirty&=~(1<<hr);
 882       }
 883     }
 884   }
 885 }
 886
 887 #ifndef NDEBUG
 888 static int host_tempreg_in_use;
 889
 890 static void host_tempreg_acquire(void)
 891 {
 892   assert(!host_tempreg_in_use);
 893   host_tempreg_in_use = 1;
 894 }
 895
 896 static void host_tempreg_release(void)
 897 {
 898   host_tempreg_in_use = 0;
 899 }
 900 #else
 901 static void host_tempreg_acquire(void) {}
 902 static void host_tempreg_release(void) {}
 903 #endif
 904
 905 #ifdef DRC_DBG
 906 extern void gen_interupt();
 907 extern void do_insn_cmp();
 908 #define FUNCNAME(f) { f, " " #f }
 909 static const struct {
 910   void *addr;
 911   const char *name;
 912 } function_names[] = {
 913   FUNCNAME(cc_interrupt),
 914   FUNCNAME(gen_interupt),
 915   FUNCNAME(get_addr_ht),
 916   FUNCNAME(get_addr),
 917   FUNCNAME(jump_handler_read8),
 918   FUNCNAME(jump_handler_read16),
 919   FUNCNAME(jump_handler_read32),
 920   FUNCNAME(jump_handler_write8),
 921   FUNCNAME(jump_handler_write16),
 922   FUNCNAME(jump_handler_write32),
 923   FUNCNAME(invalidate_addr),
 924   FUNCNAME(jump_to_new_pc),
 925   FUNCNAME(call_gteStall),
 926   FUNCNAME(new_dyna_leave),
 927   FUNCNAME(pcsx_mtc0),
 928   FUNCNAME(pcsx_mtc0_ds),
 929   FUNCNAME(do_insn_cmp),
 930 #ifdef __arm__
 931   FUNCNAME(verify_code),
 932 #endif
 933 };
 934
 935 static const char *func_name(const void *a)
 936 {
 937   int i;
 938   for (i = 0; i < sizeof(function_names)/sizeof(function_names[0]); i++)
 939     if (function_names[i].addr == a)
 940       return function_names[i].name;
 941   return "";
 942 }
 943 #else
 944 #define func_name(x) ""
 945 #endif
 946
 947 #ifdef __i386__
 948 #include "assem_x86.c"
 949 #endif
 950 #ifdef __x86_64__
 951 #include "assem_x64.c"
 952 #endif
 953 #ifdef __arm__
 954 #include "assem_arm.c"
 955 #endif
 956 #ifdef __aarch64__
 957 #include "assem_arm64.c"
 958 #endif
 959
 960 static void *get_trampoline(const void *f)
 961 {
 962   size_t i;
 963
 964   for (i = 0; i < ARRAY_SIZE(ndrc->tramp.f); i++) {
 965     if (ndrc->tramp.f[i] == f || ndrc->tramp.f[i] == NULL)
 966       break;
 967   }
 968   if (i == ARRAY_SIZE(ndrc->tramp.f)) {
 969     SysPrintf("trampoline table is full, last func %p\n", f);
 970     abort();
 971   }
 972   if (ndrc->tramp.f[i] == NULL) {
 973     start_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
 974     ndrc->tramp.f[i] = f;
 975     end_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
 976   }
 977   return &ndrc->tramp.ops[i];
 978 }
 979
 980 static void emit_far_jump(const void *f)
 981 {
 982   if (can_jump_or_call(f)) {
 983     emit_jmp(f);
 984     return;
 985   }
 986
 987   f = get_trampoline(f);
 988   emit_jmp(f);
 989 }
 990
 991 static void emit_far_call(const void *f)
 992 {
 993   if (can_jump_or_call(f)) {
 994     emit_call(f);
 995     return;
 996   }
 997
 998   f = get_trampoline(f);
 999   emit_call(f);
1000 }
1001
1002 // Add virtual address mapping to linked list
1003 void ll_add(struct ll_entry **head,int vaddr,void *addr)
1004 {
1005   struct ll_entry *new_entry;
1006   new_entry=malloc(sizeof(struct ll_entry));
1007   assert(new_entry!=NULL);
1008   new_entry->vaddr=vaddr;
1009   new_entry->reg_sv_flags=0;
1010   new_entry->addr=addr;
1011   new_entry->next=*head;
1012   *head=new_entry;
1013 }
1014
1015 void ll_add_flags(struct ll_entry **head,int vaddr,u_int reg_sv_flags,void *addr)
1016 {
1017   ll_add(head,vaddr,addr);
1018   (*head)->reg_sv_flags=reg_sv_flags;
1019 }
1020
1021 // Check if an address is already compiled
1022 // but don't return addresses which are about to expire from the cache
1023 void *check_addr(u_int vaddr)
1024 {
1025   struct ht_entry *ht_bin = hash_table_get(vaddr);
1026   size_t i;
1027   for (i = 0; i < ARRAY_SIZE(ht_bin->vaddr); i++) {
1028     if (ht_bin->vaddr[i] == vaddr)
1029       if (doesnt_expire_soon((u_char *)ht_bin->tcaddr[i] - MAX_OUTPUT_BLOCK_SIZE))
1030         if (isclean(ht_bin->tcaddr[i]))
1031           return ht_bin->tcaddr[i];
1032   }
1033   u_int page=get_page(vaddr);
1034   struct ll_entry *head;
1035   head=jump_in[page];
1036   while (head != NULL) {
1037     if (head->vaddr == vaddr) {
1038       if (doesnt_expire_soon(head->addr)) {
1039         // Update existing entry with current address
1040         if (ht_bin->vaddr[0] == vaddr) {
1041           ht_bin->tcaddr[0] = head->addr;
1042           return head->addr;
1043         }
1044         if (ht_bin->vaddr[1] == vaddr) {
1045           ht_bin->tcaddr[1] = head->addr;
1046           return head->addr;
1047         }
1048         // Insert into hash table with low priority.
1049         // Don't evict existing entries, as they are probably
1050         // addresses that are being accessed frequently.
1051         if (ht_bin->vaddr[0] == -1) {
1052           ht_bin->vaddr[0] = vaddr;
1053           ht_bin->tcaddr[0] = head->addr;
1054         }
1055         else if (ht_bin->vaddr[1] == -1) {
1056           ht_bin->vaddr[1] = vaddr;
1057           ht_bin->tcaddr[1] = head->addr;
1058         }
1059         return head->addr;
1060       }
1061     }
1062     head=head->next;
1063   }
1064   return 0;
1065 }
1066
1067 void remove_hash(int vaddr)
1068 {
1069   //printf("remove hash: %x\n",vaddr);
1070   struct ht_entry *ht_bin = hash_table_get(vaddr);
1071   if (ht_bin->vaddr[1] == vaddr) {
1072     ht_bin->vaddr[1] = -1;
1073     ht_bin->tcaddr[1] = NULL;
1074   }
1075   if (ht_bin->vaddr[0] == vaddr) {
1076     ht_bin->vaddr[0] = ht_bin->vaddr[1];
1077     ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
1078     ht_bin->vaddr[1] = -1;
1079     ht_bin->tcaddr[1] = NULL;
1080   }
1081 }
1082
1083 void ll_remove_matching_addrs(struct ll_entry **head,uintptr_t addr,int shift)
1084 {
1085   struct ll_entry *next;
1086   while(*head) {
1087     if(((uintptr_t)((*head)->addr)>>shift)==(addr>>shift) ||
1088        ((uintptr_t)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
1089     {
1090       inv_debug("EXP: Remove pointer to %p (%x)\n",(*head)->addr,(*head)->vaddr);
1091       remove_hash((*head)->vaddr);
1092       next=(*head)->next;
1093       free(*head);
1094       *head=next;
1095     }
1096     else
1097     {
1098       head=&((*head)->next);
1099     }
1100   }
1101 }
1102
1103 // Remove all entries from linked list
1104 void ll_clear(struct ll_entry **head)
1105 {
1106   struct ll_entry *cur;
1107   struct ll_entry *next;
1108   if((cur=*head)) {
1109     *head=0;
1110     while(cur) {
1111       next=cur->next;
1112       free(cur);
1113       cur=next;
1114     }
1115   }
1116 }
1117
1118 // Dereference the pointers and remove if it matches
1119 static void ll_kill_pointers(struct ll_entry *head,uintptr_t addr,int shift)
1120 {
1121   while(head) {
1122     uintptr_t ptr = (uintptr_t)get_pointer(head->addr);
1123     inv_debug("EXP: Lookup pointer to %lx at %p (%x)\n",(long)ptr,head->addr,head->vaddr);
1124     if(((ptr>>shift)==(addr>>shift)) ||
1125        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
1126     {
1127       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
1128       void *host_addr=find_extjump_insn(head->addr);
1129       mark_clear_cache(host_addr);
1130       set_jump_target(host_addr, head->addr);
1131     }
1132     head=head->next;
1133   }
1134 }
1135
1136 // This is called when we write to a compiled block (see do_invstub)
1137 static void invalidate_page(u_int page)
1138 {
1139   struct ll_entry *head;
1140   struct ll_entry *next;
1141   head=jump_in[page];
1142   jump_in[page]=0;
1143   while(head!=NULL) {
1144     inv_debug("INVALIDATE: %x\n",head->vaddr);
1145     remove_hash(head->vaddr);
1146     next=head->next;
1147     free(head);
1148     head=next;
1149   }
1150   head=jump_out[page];
1151   jump_out[page]=0;
1152   while(head!=NULL) {
1153     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
1154     void *host_addr=find_extjump_insn(head->addr);
1155     mark_clear_cache(host_addr);
1156     set_jump_target(host_addr, head->addr);
1157     next=head->next;
1158     free(head);
1159     head=next;
1160   }
1161 }
1162
1163 static void invalidate_block_range(u_int block, u_int first, u_int last)
1164 {
1165   u_int page=get_page(block<<12);
1166   //printf("first=%d last=%d\n",first,last);
1167   invalidate_page(page);
1168   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1169   assert(last<page+5);
1170   // Invalidate the adjacent pages if a block crosses a 4K boundary
1171   while(first<page) {
1172     invalidate_page(first);
1173     first++;
1174   }
1175   for(first=page+1;first<last;first++) {
1176     invalidate_page(first);
1177   }
1178   do_clear_cache();
1179
1180   // Don't trap writes
1181   invalid_code[block]=1;
1182
1183   #ifdef USE_MINI_HT
1184   memset(mini_ht,-1,sizeof(mini_ht));
1185   #endif
1186 }
1187
1188 void invalidate_block(u_int block)
1189 {
1190   u_int page=get_page(block<<12);
1191   u_int vpage=get_vpage(block<<12);
1192   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1193   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1194   u_int first,last;
1195   first=last=page;
1196   struct ll_entry *head;
1197   head=jump_dirty[vpage];
1198   //printf("page=%d vpage=%d\n",page,vpage);
1199   while(head!=NULL) {
1200     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1201       u_char *start, *end;
1202       get_bounds(head->addr, &start, &end);
1203       //printf("start: %p end: %p\n", start, end);
1204       if (page < 2048 && start >= rdram && end < rdram+RAM_SIZE) {
1205         if (((start-rdram)>>12) <= page && ((end-1-rdram)>>12) >= page) {
1206           if ((((start-rdram)>>12)&2047) < first) first = ((start-rdram)>>12)&2047;
1207           if ((((end-1-rdram)>>12)&2047) > last)  last = ((end-1-rdram)>>12)&2047;
1208         }
1209       }
1210     }
1211     head=head->next;
1212   }
1213   invalidate_block_range(block,first,last);
1214 }
1215
1216 void invalidate_addr(u_int addr)
1217 {
1218   //static int rhits;
1219   // this check is done by the caller
1220   //if (inv_code_start<=addr&&addr<=inv_code_end) { rhits++; return; }
1221   u_int page=get_vpage(addr);
1222   if(page<2048) { // RAM
1223     struct ll_entry *head;
1224     u_int addr_min=~0, addr_max=0;
1225     u_int mask=RAM_SIZE-1;
1226     u_int addr_main=0x80000000|(addr&mask);
1227     int pg1;
1228     inv_code_start=addr_main&~0xfff;
1229     inv_code_end=addr_main|0xfff;
1230     pg1=page;
1231     if (pg1>0) {
1232       // must check previous page too because of spans..
1233       pg1--;
1234       inv_code_start-=0x1000;
1235     }
1236     for(;pg1<=page;pg1++) {
1237       for(head=jump_dirty[pg1];head!=NULL;head=head->next) {
1238         u_char *start_h, *end_h;
1239         u_int start, end;
1240         get_bounds(head->addr, &start_h, &end_h);
1241         start = (uintptr_t)start_h - ram_offset;
1242         end = (uintptr_t)end_h - ram_offset;
1243         if(start<=addr_main&&addr_main<end) {
1244           if(start<addr_min) addr_min=start;
1245           if(end>addr_max) addr_max=end;
1246         }
1247         else if(addr_main<start) {
1248           if(start<inv_code_end)
1249             inv_code_end=start-1;
1250         }
1251         else {
1252           if(end>inv_code_start)
1253             inv_code_start=end;
1254         }
1255       }
1256     }
1257     if (addr_min!=~0) {
1258       inv_debug("INV ADDR: %08x hit %08x-%08x\n", addr, addr_min, addr_max);
1259       inv_code_start=inv_code_end=~0;
1260       invalidate_block_range(addr>>12,(addr_min&mask)>>12,(addr_max&mask)>>12);
1261       return;
1262     }
1263     else {
1264       inv_code_start=(addr&~mask)|(inv_code_start&mask);
1265       inv_code_end=(addr&~mask)|(inv_code_end&mask);
1266       inv_debug("INV ADDR: %08x miss, inv %08x-%08x, sk %d\n", addr, inv_code_start, inv_code_end, 0);
1267       return;
1268     }
1269   }
1270   invalidate_block(addr>>12);
1271 }
1272
1273 // This is called when loading a save state.
1274 // Anything could have changed, so invalidate everything.
1275 void invalidate_all_pages(void)
1276 {
1277   u_int page;
1278   for(page=0;page<4096;page++)
1279     invalidate_page(page);
1280   for(page=0;page<1048576;page++)
1281     if(!invalid_code[page]) {
1282       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1283       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1284     }
1285   #ifdef USE_MINI_HT
1286   memset(mini_ht,-1,sizeof(mini_ht));
1287   #endif
1288   do_clear_cache();
1289 }
1290
1291 static void do_invstub(int n)
1292 {
1293   literal_pool(20);
1294   u_int reglist=stubs[n].a;
1295   set_jump_target(stubs[n].addr, out);
1296   save_regs(reglist);
1297   if(stubs[n].b!=0) emit_mov(stubs[n].b,0);
1298   emit_far_call(invalidate_addr);
1299   restore_regs(reglist);
1300   emit_jmp(stubs[n].retaddr); // return address
1301 }
1302
1303 // Add an entry to jump_out after making a link
1304 // src should point to code by emit_extjump2()
1305 void add_link(u_int vaddr,void *src)
1306 {
1307   u_int page=get_page(vaddr);
1308   inv_debug("add_link: %p -> %x (%d)\n",src,vaddr,page);
1309   check_extjump2(src);
1310   ll_add(jump_out+page,vaddr,src);
1311   //void *ptr=get_pointer(src);
1312   //inv_debug("add_link: Pointer is to %p\n",ptr);
1313 }
1314
1315 // If a code block was found to be unmodified (bit was set in
1316 // restore_candidate) and it remains unmodified (bit is clear
1317 // in invalid_code) then move the entries for that 4K page from
1318 // the dirty list to the clean list.
1319 void clean_blocks(u_int page)
1320 {
1321   struct ll_entry *head;
1322   inv_debug("INV: clean_blocks page=%d\n",page);
1323   head=jump_dirty[page];
1324   while(head!=NULL) {
1325     if(!invalid_code[head->vaddr>>12]) {
1326       // Don't restore blocks which are about to expire from the cache
1327       if (doesnt_expire_soon(head->addr)) {
1328         if(verify_dirty(head->addr)) {
1329           u_char *start, *end;
1330           //printf("Possibly Restore %x (%p)\n",head->vaddr, head->addr);
1331           u_int i;
1332           u_int inv=0;
1333           get_bounds(head->addr, &start, &end);
1334           if (start - rdram < RAM_SIZE) {
1335             for (i = (start-rdram+0x80000000)>>12; i <= (end-1-rdram+0x80000000)>>12; i++) {
1336               inv|=invalid_code[i];
1337             }
1338           }
1339           else if((signed int)head->vaddr>=(signed int)0x80000000+RAM_SIZE) {
1340             inv=1;
1341           }
1342           if(!inv) {
1343             void *clean_addr = get_clean_addr(head->addr);
1344             if (doesnt_expire_soon(clean_addr)) {
1345               u_int ppage=page;
1346               inv_debug("INV: Restored %x (%p/%p)\n",head->vaddr, head->addr, clean_addr);
1347               //printf("page=%x, addr=%x\n",page,head->vaddr);
1348               //assert(head->vaddr>>12==(page|0x80000));
1349               ll_add_flags(jump_in+ppage,head->vaddr,head->reg_sv_flags,clean_addr);
1350               struct ht_entry *ht_bin = hash_table_get(head->vaddr);
1351               if (ht_bin->vaddr[0] == head->vaddr)
1352                 ht_bin->tcaddr[0] = clean_addr; // Replace existing entry
1353               if (ht_bin->vaddr[1] == head->vaddr)
1354                 ht_bin->tcaddr[1] = clean_addr; // Replace existing entry
1355             }
1356           }
1357         }
1358       }
1359     }
1360     head=head->next;
1361   }
1362 }
1363
1364 /* Register allocation */
1365
1366 // Note: registers are allocated clean (unmodified state)
1367 // if you intend to modify the register, you must call dirty_reg().
1368 static void alloc_reg(struct regstat *cur,int i,signed char reg)
1369 {
1370   int r,hr;
1371   int preferred_reg = (reg&7);
1372   if(reg==CCREG) preferred_reg=HOST_CCREG;
1373   if(reg==PTEMP||reg==FTEMP) preferred_reg=12;
1374
1375   // Don't allocate unused registers
1376   if((cur->u>>reg)&1) return;
1377
1378   // see if it's already allocated
1379   for(hr=0;hr<HOST_REGS;hr++)
1380   {
1381     if(cur->regmap[hr]==reg) return;
1382   }
1383
1384   // Keep the same mapping if the register was already allocated in a loop
1385   preferred_reg = loop_reg(i,reg,preferred_reg);
1386
1387   // Try to allocate the preferred register
1388   if(cur->regmap[preferred_reg]==-1) {
1389     cur->regmap[preferred_reg]=reg;
1390     cur->dirty&=~(1<<preferred_reg);
1391     cur->isconst&=~(1<<preferred_reg);
1392     return;
1393   }
1394   r=cur->regmap[preferred_reg];
1395   assert(r < 64);
1396   if((cur->u>>r)&1) {
1397     cur->regmap[preferred_reg]=reg;
1398     cur->dirty&=~(1<<preferred_reg);
1399     cur->isconst&=~(1<<preferred_reg);
1400     return;
1401   }
1402
1403   // Clear any unneeded registers
1404   // We try to keep the mapping consistent, if possible, because it
1405   // makes branches easier (especially loops).  So we try to allocate
1406   // first (see above) before removing old mappings.  If this is not
1407   // possible then go ahead and clear out the registers that are no
1408   // longer needed.
1409   for(hr=0;hr<HOST_REGS;hr++)
1410   {
1411     r=cur->regmap[hr];
1412     if(r>=0) {
1413       assert(r < 64);
1414       if((cur->u>>r)&1) {cur->regmap[hr]=-1;break;}
1415     }
1416   }
1417   // Try to allocate any available register, but prefer
1418   // registers that have not been used recently.
1419   if(i>0) {
1420     for(hr=0;hr<HOST_REGS;hr++) {
1421       if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1422         if(regs[i-1].regmap[hr]!=rs1[i-1]&&regs[i-1].regmap[hr]!=rs2[i-1]&&regs[i-1].regmap[hr]!=rt1[i-1]&&regs[i-1].regmap[hr]!=rt2[i-1]) {
1423           cur->regmap[hr]=reg;
1424           cur->dirty&=~(1<<hr);
1425           cur->isconst&=~(1<<hr);
1426           return;
1427         }
1428       }
1429     }
1430   }
1431   // Try to allocate any available register
1432   for(hr=0;hr<HOST_REGS;hr++) {
1433     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1434       cur->regmap[hr]=reg;
1435       cur->dirty&=~(1<<hr);
1436       cur->isconst&=~(1<<hr);
1437       return;
1438     }
1439   }
1440
1441   // Ok, now we have to evict someone
1442   // Pick a register we hopefully won't need soon
1443   u_char hsn[MAXREG+1];
1444   memset(hsn,10,sizeof(hsn));
1445   int j;
1446   lsn(hsn,i,&preferred_reg);
1447   //printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",cur->regmap[0],cur->regmap[1],cur->regmap[2],cur->regmap[3],cur->regmap[5],cur->regmap[6],cur->regmap[7]);
1448   //printf("hsn(%x): %d %d %d %d %d %d %d\n",start+i*4,hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1449   if(i>0) {
1450     // Don't evict the cycle count at entry points, otherwise the entry
1451     // stub will have to write it.
1452     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1453     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1454     for(j=10;j>=3;j--)
1455     {
1456       // Alloc preferred register if available
1457       if(hsn[r=cur->regmap[preferred_reg]&63]==j) {
1458         for(hr=0;hr<HOST_REGS;hr++) {
1459           // Evict both parts of a 64-bit register
1460           if((cur->regmap[hr]&63)==r) {
1461             cur->regmap[hr]=-1;
1462             cur->dirty&=~(1<<hr);
1463             cur->isconst&=~(1<<hr);
1464           }
1465         }
1466         cur->regmap[preferred_reg]=reg;
1467         return;
1468       }
1469       for(r=1;r<=MAXREG;r++)
1470       {
1471         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1472           for(hr=0;hr<HOST_REGS;hr++) {
1473             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1474               if(cur->regmap[hr]==r) {
1475                 cur->regmap[hr]=reg;
1476                 cur->dirty&=~(1<<hr);
1477                 cur->isconst&=~(1<<hr);
1478                 return;
1479               }
1480             }
1481           }
1482         }
1483       }
1484     }
1485   }
1486   for(j=10;j>=0;j--)
1487   {
1488     for(r=1;r<=MAXREG;r++)
1489     {
1490       if(hsn[r]==j) {
1491         for(hr=0;hr<HOST_REGS;hr++) {
1492           if(cur->regmap[hr]==r) {
1493             cur->regmap[hr]=reg;
1494             cur->dirty&=~(1<<hr);
1495             cur->isconst&=~(1<<hr);
1496             return;
1497           }
1498         }
1499       }
1500     }
1501   }
1502   SysPrintf("This shouldn't happen (alloc_reg)");abort();
1503 }
1504
1505 // Allocate a temporary register.  This is done without regard to
1506 // dirty status or whether the register we request is on the unneeded list
1507 // Note: This will only allocate one register, even if called multiple times
1508 static void alloc_reg_temp(struct regstat *cur,int i,signed char reg)
1509 {
1510   int r,hr;
1511   int preferred_reg = -1;
1512
1513   // see if it's already allocated
1514   for(hr=0;hr<HOST_REGS;hr++)
1515   {
1516     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==reg) return;
1517   }
1518
1519   // Try to allocate any available register
1520   for(hr=HOST_REGS-1;hr>=0;hr--) {
1521     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1522       cur->regmap[hr]=reg;
1523       cur->dirty&=~(1<<hr);
1524       cur->isconst&=~(1<<hr);
1525       return;
1526     }
1527   }
1528
1529   // Find an unneeded register
1530   for(hr=HOST_REGS-1;hr>=0;hr--)
1531   {
1532     r=cur->regmap[hr];
1533     if(r>=0) {
1534       assert(r < 64);
1535       if((cur->u>>r)&1) {
1536         if(i==0||((unneeded_reg[i-1]>>r)&1)) {
1537           cur->regmap[hr]=reg;
1538           cur->dirty&=~(1<<hr);
1539           cur->isconst&=~(1<<hr);
1540           return;
1541         }
1542       }
1543     }
1544   }
1545
1546   // Ok, now we have to evict someone
1547   // Pick a register we hopefully won't need soon
1548   // TODO: we might want to follow unconditional jumps here
1549   // TODO: get rid of dupe code and make this into a function
1550   u_char hsn[MAXREG+1];
1551   memset(hsn,10,sizeof(hsn));
1552   int j;
1553   lsn(hsn,i,&preferred_reg);
1554   //printf("hsn: %d %d %d %d %d %d %d\n",hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1555   if(i>0) {
1556     // Don't evict the cycle count at entry points, otherwise the entry
1557     // stub will have to write it.
1558     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1559     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1560     for(j=10;j>=3;j--)
1561     {
1562       for(r=1;r<=MAXREG;r++)
1563       {
1564         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1565           for(hr=0;hr<HOST_REGS;hr++) {
1566             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1567               if(cur->regmap[hr]==r) {
1568                 cur->regmap[hr]=reg;
1569                 cur->dirty&=~(1<<hr);
1570                 cur->isconst&=~(1<<hr);
1571                 return;
1572               }
1573             }
1574           }
1575         }
1576       }
1577     }
1578   }
1579   for(j=10;j>=0;j--)
1580   {
1581     for(r=1;r<=MAXREG;r++)
1582     {
1583       if(hsn[r]==j) {
1584         for(hr=0;hr<HOST_REGS;hr++) {
1585           if(cur->regmap[hr]==r) {
1586             cur->regmap[hr]=reg;
1587             cur->dirty&=~(1<<hr);
1588             cur->isconst&=~(1<<hr);
1589             return;
1590           }
1591         }
1592       }
1593     }
1594   }
1595   SysPrintf("This shouldn't happen");abort();
1596 }
1597
1598 static void mov_alloc(struct regstat *current,int i)
1599 {
1600   // Note: Don't need to actually alloc the source registers
1601   //alloc_reg(current,i,rs1[i]);
1602   alloc_reg(current,i,rt1[i]);
1603
1604   clear_const(current,rs1[i]);
1605   clear_const(current,rt1[i]);
1606   dirty_reg(current,rt1[i]);
1607 }
1608
1609 static void shiftimm_alloc(struct regstat *current,int i)
1610 {
1611   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1612   {
1613     if(rt1[i]) {
1614       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1615       else lt1[i]=rs1[i];
1616       alloc_reg(current,i,rt1[i]);
1617       dirty_reg(current,rt1[i]);
1618       if(is_const(current,rs1[i])) {
1619         int v=get_const(current,rs1[i]);
1620         if(opcode2[i]==0x00) set_const(current,rt1[i],v<<imm[i]);
1621         if(opcode2[i]==0x02) set_const(current,rt1[i],(u_int)v>>imm[i]);
1622         if(opcode2[i]==0x03) set_const(current,rt1[i],v>>imm[i]);
1623       }
1624       else clear_const(current,rt1[i]);
1625     }
1626   }
1627   else
1628   {
1629     clear_const(current,rs1[i]);
1630     clear_const(current,rt1[i]);
1631   }
1632
1633   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1634   {
1635     assert(0);
1636   }
1637   if(opcode2[i]==0x3c) // DSLL32
1638   {
1639     assert(0);
1640   }
1641   if(opcode2[i]==0x3e) // DSRL32
1642   {
1643     assert(0);
1644   }
1645   if(opcode2[i]==0x3f) // DSRA32
1646   {
1647     assert(0);
1648   }
1649 }
1650
1651 static void shift_alloc(struct regstat *current,int i)
1652 {
1653   if(rt1[i]) {
1654     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1655     {
1656       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1657       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1658       alloc_reg(current,i,rt1[i]);
1659       if(rt1[i]==rs2[i]) {
1660         alloc_reg_temp(current,i,-1);
1661         minimum_free_regs[i]=1;
1662       }
1663     } else { // DSLLV/DSRLV/DSRAV
1664       assert(0);
1665     }
1666     clear_const(current,rs1[i]);
1667     clear_const(current,rs2[i]);
1668     clear_const(current,rt1[i]);
1669     dirty_reg(current,rt1[i]);
1670   }
1671 }
1672
1673 static void alu_alloc(struct regstat *current,int i)
1674 {
1675   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1676     if(rt1[i]) {
1677       if(rs1[i]&&rs2[i]) {
1678         alloc_reg(current,i,rs1[i]);
1679         alloc_reg(current,i,rs2[i]);
1680       }
1681       else {
1682         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1683         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1684       }
1685       alloc_reg(current,i,rt1[i]);
1686     }
1687   }
1688   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1689     if(rt1[i]) {
1690       alloc_reg(current,i,rs1[i]);
1691       alloc_reg(current,i,rs2[i]);
1692       alloc_reg(current,i,rt1[i]);
1693     }
1694   }
1695   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1696     if(rt1[i]) {
1697       if(rs1[i]&&rs2[i]) {
1698         alloc_reg(current,i,rs1[i]);
1699         alloc_reg(current,i,rs2[i]);
1700       }
1701       else
1702       {
1703         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1704         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1705       }
1706       alloc_reg(current,i,rt1[i]);
1707     }
1708   }
1709   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1710     assert(0);
1711   }
1712   clear_const(current,rs1[i]);
1713   clear_const(current,rs2[i]);
1714   clear_const(current,rt1[i]);
1715   dirty_reg(current,rt1[i]);
1716 }
1717
1718 static void imm16_alloc(struct regstat *current,int i)
1719 {
1720   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1721   else lt1[i]=rs1[i];
1722   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1723   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1724     assert(0);
1725   }
1726   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1727     clear_const(current,rs1[i]);
1728     clear_const(current,rt1[i]);
1729   }
1730   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1731     if(is_const(current,rs1[i])) {
1732       int v=get_const(current,rs1[i]);
1733       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1734       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1735       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1736     }
1737     else clear_const(current,rt1[i]);
1738   }
1739   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1740     if(is_const(current,rs1[i])) {
1741       int v=get_const(current,rs1[i]);
1742       set_const(current,rt1[i],v+imm[i]);
1743     }
1744     else clear_const(current,rt1[i]);
1745   }
1746   else {
1747     set_const(current,rt1[i],imm[i]<<16); // LUI
1748   }
1749   dirty_reg(current,rt1[i]);
1750 }
1751
1752 static void load_alloc(struct regstat *current,int i)
1753 {
1754   clear_const(current,rt1[i]);
1755   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1756   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1757   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1758   if(rt1[i]&&!((current->u>>rt1[i])&1)) {
1759     alloc_reg(current,i,rt1[i]);
1760     assert(get_reg(current->regmap,rt1[i])>=0);
1761     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1762     {
1763       assert(0);
1764     }
1765     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1766     {
1767       assert(0);
1768     }
1769     dirty_reg(current,rt1[i]);
1770     // LWL/LWR need a temporary register for the old value
1771     if(opcode[i]==0x22||opcode[i]==0x26)
1772     {
1773       alloc_reg(current,i,FTEMP);
1774       alloc_reg_temp(current,i,-1);
1775       minimum_free_regs[i]=1;
1776     }
1777   }
1778   else
1779   {
1780     // Load to r0 or unneeded register (dummy load)
1781     // but we still need a register to calculate the address
1782     if(opcode[i]==0x22||opcode[i]==0x26)
1783     {
1784       alloc_reg(current,i,FTEMP); // LWL/LWR need another temporary
1785     }
1786     alloc_reg_temp(current,i,-1);
1787     minimum_free_regs[i]=1;
1788     if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1789     {
1790       assert(0);
1791     }
1792   }
1793 }
1794
1795 void store_alloc(struct regstat *current,int i)
1796 {
1797   clear_const(current,rs2[i]);
1798   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1799   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1800   alloc_reg(current,i,rs2[i]);
1801   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1802     assert(0);
1803   }
1804   #if defined(HOST_IMM8)
1805   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1806   else alloc_reg(current,i,INVCP);
1807   #endif
1808   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) { // SWL/SWL/SDL/SDR
1809     alloc_reg(current,i,FTEMP);
1810   }
1811   // We need a temporary register for address generation
1812   alloc_reg_temp(current,i,-1);
1813   minimum_free_regs[i]=1;
1814 }
1815
1816 void c1ls_alloc(struct regstat *current,int i)
1817 {
1818   //clear_const(current,rs1[i]); // FIXME
1819   clear_const(current,rt1[i]);
1820   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1821   alloc_reg(current,i,CSREG); // Status
1822   alloc_reg(current,i,FTEMP);
1823   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1824     assert(0);
1825   }
1826   #if defined(HOST_IMM8)
1827   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1828   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1829     alloc_reg(current,i,INVCP);
1830   #endif
1831   // We need a temporary register for address generation
1832   alloc_reg_temp(current,i,-1);
1833 }
1834
1835 void c2ls_alloc(struct regstat *current,int i)
1836 {
1837   clear_const(current,rt1[i]);
1838   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1839   alloc_reg(current,i,FTEMP);
1840   #if defined(HOST_IMM8)
1841   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1842   if((opcode[i]&0x3b)==0x3a) // SWC2/SDC2
1843     alloc_reg(current,i,INVCP);
1844   #endif
1845   // We need a temporary register for address generation
1846   alloc_reg_temp(current,i,-1);
1847   minimum_free_regs[i]=1;
1848 }
1849
1850 #ifndef multdiv_alloc
1851 void multdiv_alloc(struct regstat *current,int i)
1852 {
1853   //  case 0x18: MULT
1854   //  case 0x19: MULTU
1855   //  case 0x1A: DIV
1856   //  case 0x1B: DIVU
1857   //  case 0x1C: DMULT
1858   //  case 0x1D: DMULTU
1859   //  case 0x1E: DDIV
1860   //  case 0x1F: DDIVU
1861   clear_const(current,rs1[i]);
1862   clear_const(current,rs2[i]);
1863   if(rs1[i]&&rs2[i])
1864   {
1865     if((opcode2[i]&4)==0) // 32-bit
1866     {
1867       current->u&=~(1LL<<HIREG);
1868       current->u&=~(1LL<<LOREG);
1869       alloc_reg(current,i,HIREG);
1870       alloc_reg(current,i,LOREG);
1871       alloc_reg(current,i,rs1[i]);
1872       alloc_reg(current,i,rs2[i]);
1873       dirty_reg(current,HIREG);
1874       dirty_reg(current,LOREG);
1875     }
1876     else // 64-bit
1877     {
1878       assert(0);
1879     }
1880   }
1881   else
1882   {
1883     // Multiply by zero is zero.
1884     // MIPS does not have a divide by zero exception.
1885     // The result is undefined, we return zero.
1886     alloc_reg(current,i,HIREG);
1887     alloc_reg(current,i,LOREG);
1888     dirty_reg(current,HIREG);
1889     dirty_reg(current,LOREG);
1890   }
1891 }
1892 #endif
1893
1894 void cop0_alloc(struct regstat *current,int i)
1895 {
1896   if(opcode2[i]==0) // MFC0
1897   {
1898     if(rt1[i]) {
1899       clear_const(current,rt1[i]);
1900       alloc_all(current,i);
1901       alloc_reg(current,i,rt1[i]);
1902       dirty_reg(current,rt1[i]);
1903     }
1904   }
1905   else if(opcode2[i]==4) // MTC0
1906   {
1907     if(rs1[i]){
1908       clear_const(current,rs1[i]);
1909       alloc_reg(current,i,rs1[i]);
1910       alloc_all(current,i);
1911     }
1912     else {
1913       alloc_all(current,i); // FIXME: Keep r0
1914       current->u&=~1LL;
1915       alloc_reg(current,i,0);
1916     }
1917   }
1918   else
1919   {
1920     // TLBR/TLBWI/TLBWR/TLBP/ERET
1921     assert(opcode2[i]==0x10);
1922     alloc_all(current,i);
1923   }
1924   minimum_free_regs[i]=HOST_REGS;
1925 }
1926
1927 static void cop2_alloc(struct regstat *current,int i)
1928 {
1929   if (opcode2[i] < 3) // MFC2/CFC2
1930   {
1931     alloc_cc(current,i); // for stalls
1932     dirty_reg(current,CCREG);
1933     if(rt1[i]){
1934       clear_const(current,rt1[i]);
1935       alloc_reg(current,i,rt1[i]);
1936       dirty_reg(current,rt1[i]);
1937     }
1938   }
1939   else if (opcode2[i] > 3) // MTC2/CTC2
1940   {
1941     if(rs1[i]){
1942       clear_const(current,rs1[i]);
1943       alloc_reg(current,i,rs1[i]);
1944     }
1945     else {
1946       current->u&=~1LL;
1947       alloc_reg(current,i,0);
1948     }
1949   }
1950   alloc_reg_temp(current,i,-1);
1951   minimum_free_regs[i]=1;
1952 }
1953
1954 void c2op_alloc(struct regstat *current,int i)
1955 {
1956   alloc_cc(current,i); // for stalls
1957   dirty_reg(current,CCREG);
1958   alloc_reg_temp(current,i,-1);
1959 }
1960
1961 void syscall_alloc(struct regstat *current,int i)
1962 {
1963   alloc_cc(current,i);
1964   dirty_reg(current,CCREG);
1965   alloc_all(current,i);
1966   minimum_free_regs[i]=HOST_REGS;
1967   current->isconst=0;
1968 }
1969
1970 void delayslot_alloc(struct regstat *current,int i)
1971 {
1972   switch(itype[i]) {
1973     case UJUMP:
1974     case CJUMP:
1975     case SJUMP:
1976     case RJUMP:
1977     case SYSCALL:
1978     case HLECALL:
1979     case SPAN:
1980       assem_debug("jump in the delay slot.  this shouldn't happen.\n");//abort();
1981       SysPrintf("Disabled speculative precompilation\n");
1982       stop_after_jal=1;
1983       break;
1984     case IMM16:
1985       imm16_alloc(current,i);
1986       break;
1987     case LOAD:
1988     case LOADLR:
1989       load_alloc(current,i);
1990       break;
1991     case STORE:
1992     case STORELR:
1993       store_alloc(current,i);
1994       break;
1995     case ALU:
1996       alu_alloc(current,i);
1997       break;
1998     case SHIFT:
1999       shift_alloc(current,i);
2000       break;
2001     case MULTDIV:
2002       multdiv_alloc(current,i);
2003       break;
2004     case SHIFTIMM:
2005       shiftimm_alloc(current,i);
2006       break;
2007     case MOV:
2008       mov_alloc(current,i);
2009       break;
2010     case COP0:
2011       cop0_alloc(current,i);
2012       break;
2013     case COP1:
2014       break;
2015     case COP2:
2016       cop2_alloc(current,i);
2017       break;
2018     case C1LS:
2019       c1ls_alloc(current,i);
2020       break;
2021     case C2LS:
2022       c2ls_alloc(current,i);
2023       break;
2024     case C2OP:
2025       c2op_alloc(current,i);
2026       break;
2027   }
2028 }
2029
2030 // Special case where a branch and delay slot span two pages in virtual memory
2031 static void pagespan_alloc(struct regstat *current,int i)
2032 {
2033   current->isconst=0;
2034   current->wasconst=0;
2035   regs[i].wasconst=0;
2036   minimum_free_regs[i]=HOST_REGS;
2037   alloc_all(current,i);
2038   alloc_cc(current,i);
2039   dirty_reg(current,CCREG);
2040   if(opcode[i]==3) // JAL
2041   {
2042     alloc_reg(current,i,31);
2043     dirty_reg(current,31);
2044   }
2045   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
2046   {
2047     alloc_reg(current,i,rs1[i]);
2048     if (rt1[i]!=0) {
2049       alloc_reg(current,i,rt1[i]);
2050       dirty_reg(current,rt1[i]);
2051     }
2052   }
2053   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
2054   {
2055     if(rs1[i]) alloc_reg(current,i,rs1[i]);
2056     if(rs2[i]) alloc_reg(current,i,rs2[i]);
2057   }
2058   else
2059   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
2060   {
2061     if(rs1[i]) alloc_reg(current,i,rs1[i]);
2062   }
2063   //else ...
2064 }
2065
2066 static void add_stub(enum stub_type type, void *addr, void *retaddr,
2067   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e)
2068 {
2069   assert(stubcount < ARRAY_SIZE(stubs));
2070   stubs[stubcount].type = type;
2071   stubs[stubcount].addr = addr;
2072   stubs[stubcount].retaddr = retaddr;
2073   stubs[stubcount].a = a;
2074   stubs[stubcount].b = b;
2075   stubs[stubcount].c = c;
2076   stubs[stubcount].d = d;
2077   stubs[stubcount].e = e;
2078   stubcount++;
2079 }
2080
2081 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
2082   int i, int addr_reg, const struct regstat *i_regs, int ccadj, u_int reglist)
2083 {
2084   add_stub(type, addr, retaddr, i, addr_reg, (uintptr_t)i_regs, ccadj, reglist);
2085 }
2086
2087 // Write out a single register
2088 static void wb_register(signed char r,signed char regmap[],uint64_t dirty)
2089 {
2090   int hr;
2091   for(hr=0;hr<HOST_REGS;hr++) {
2092     if(hr!=EXCLUDE_REG) {
2093       if((regmap[hr]&63)==r) {
2094         if((dirty>>hr)&1) {
2095           assert(regmap[hr]<64);
2096           emit_storereg(r,hr);
2097         }
2098       }
2099     }
2100   }
2101 }
2102
2103 static void wb_valid(signed char pre[],signed char entry[],u_int dirty_pre,u_int dirty,uint64_t u)
2104 {
2105   //if(dirty_pre==dirty) return;
2106   int hr,reg;
2107   for(hr=0;hr<HOST_REGS;hr++) {
2108     if(hr!=EXCLUDE_REG) {
2109       reg=pre[hr];
2110       if(((~u)>>(reg&63))&1) {
2111         if(reg>0) {
2112           if(((dirty_pre&~dirty)>>hr)&1) {
2113             if(reg>0&&reg<34) {
2114               emit_storereg(reg,hr);
2115             }
2116             else if(reg>=64) {
2117               assert(0);
2118             }
2119           }
2120         }
2121       }
2122     }
2123   }
2124 }
2125
2126 // trashes r2
2127 static void pass_args(int a0, int a1)
2128 {
2129   if(a0==1&&a1==0) {
2130     // must swap
2131     emit_mov(a0,2); emit_mov(a1,1); emit_mov(2,0);
2132   }
2133   else if(a0!=0&&a1==0) {
2134     emit_mov(a1,1);
2135     if (a0>=0) emit_mov(a0,0);
2136   }
2137   else {
2138     if(a0>=0&&a0!=0) emit_mov(a0,0);
2139     if(a1>=0&&a1!=1) emit_mov(a1,1);
2140   }
2141 }
2142
2143 static void alu_assemble(int i,struct regstat *i_regs)
2144 {
2145   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2146     if(rt1[i]) {
2147       signed char s1,s2,t;
2148       t=get_reg(i_regs->regmap,rt1[i]);
2149       if(t>=0) {
2150         s1=get_reg(i_regs->regmap,rs1[i]);
2151         s2=get_reg(i_regs->regmap,rs2[i]);
2152         if(rs1[i]&&rs2[i]) {
2153           assert(s1>=0);
2154           assert(s2>=0);
2155           if(opcode2[i]&2) emit_sub(s1,s2,t);
2156           else emit_add(s1,s2,t);
2157         }
2158         else if(rs1[i]) {
2159           if(s1>=0) emit_mov(s1,t);
2160           else emit_loadreg(rs1[i],t);
2161         }
2162         else if(rs2[i]) {
2163           if(s2>=0) {
2164             if(opcode2[i]&2) emit_neg(s2,t);
2165             else emit_mov(s2,t);
2166           }
2167           else {
2168             emit_loadreg(rs2[i],t);
2169             if(opcode2[i]&2) emit_neg(t,t);
2170           }
2171         }
2172         else emit_zeroreg(t);
2173       }
2174     }
2175   }
2176   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2177     assert(0);
2178   }
2179   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2180     if(rt1[i]) {
2181       signed char s1l,s2l,t;
2182       {
2183         t=get_reg(i_regs->regmap,rt1[i]);
2184         //assert(t>=0);
2185         if(t>=0) {
2186           s1l=get_reg(i_regs->regmap,rs1[i]);
2187           s2l=get_reg(i_regs->regmap,rs2[i]);
2188           if(rs2[i]==0) // rx<r0
2189           {
2190             if(opcode2[i]==0x2a&&rs1[i]!=0) { // SLT
2191               assert(s1l>=0);
2192               emit_shrimm(s1l,31,t);
2193             }
2194             else // SLTU (unsigned can not be less than zero, 0<0)
2195               emit_zeroreg(t);
2196           }
2197           else if(rs1[i]==0) // r0<rx
2198           {
2199             assert(s2l>=0);
2200             if(opcode2[i]==0x2a) // SLT
2201               emit_set_gz32(s2l,t);
2202             else // SLTU (set if not zero)
2203               emit_set_nz32(s2l,t);
2204           }
2205           else{
2206             assert(s1l>=0);assert(s2l>=0);
2207             if(opcode2[i]==0x2a) // SLT
2208               emit_set_if_less32(s1l,s2l,t);
2209             else // SLTU
2210               emit_set_if_carry32(s1l,s2l,t);
2211           }
2212         }
2213       }
2214     }
2215   }
2216   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2217     if(rt1[i]) {
2218       signed char s1l,s2l,tl;
2219       tl=get_reg(i_regs->regmap,rt1[i]);
2220       {
2221         if(tl>=0) {
2222           s1l=get_reg(i_regs->regmap,rs1[i]);
2223           s2l=get_reg(i_regs->regmap,rs2[i]);
2224           if(rs1[i]&&rs2[i]) {
2225             assert(s1l>=0);
2226             assert(s2l>=0);
2227             if(opcode2[i]==0x24) { // AND
2228               emit_and(s1l,s2l,tl);
2229             } else
2230             if(opcode2[i]==0x25) { // OR
2231               emit_or(s1l,s2l,tl);
2232             } else
2233             if(opcode2[i]==0x26) { // XOR
2234               emit_xor(s1l,s2l,tl);
2235             } else
2236             if(opcode2[i]==0x27) { // NOR
2237               emit_or(s1l,s2l,tl);
2238               emit_not(tl,tl);
2239             }
2240           }
2241           else
2242           {
2243             if(opcode2[i]==0x24) { // AND
2244               emit_zeroreg(tl);
2245             } else
2246             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2247               if(rs1[i]){
2248                 if(s1l>=0) emit_mov(s1l,tl);
2249                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2250               }
2251               else
2252               if(rs2[i]){
2253                 if(s2l>=0) emit_mov(s2l,tl);
2254                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2255               }
2256               else emit_zeroreg(tl);
2257             } else
2258             if(opcode2[i]==0x27) { // NOR
2259               if(rs1[i]){
2260                 if(s1l>=0) emit_not(s1l,tl);
2261                 else {
2262                   emit_loadreg(rs1[i],tl);
2263                   emit_not(tl,tl);
2264                 }
2265               }
2266               else
2267               if(rs2[i]){
2268                 if(s2l>=0) emit_not(s2l,tl);
2269                 else {
2270                   emit_loadreg(rs2[i],tl);
2271                   emit_not(tl,tl);
2272                 }
2273               }
2274               else emit_movimm(-1,tl);
2275             }
2276           }
2277         }
2278       }
2279     }
2280   }
2281 }
2282
2283 void imm16_assemble(int i,struct regstat *i_regs)
2284 {
2285   if (opcode[i]==0x0f) { // LUI
2286     if(rt1[i]) {
2287       signed char t;
2288       t=get_reg(i_regs->regmap,rt1[i]);
2289       //assert(t>=0);
2290       if(t>=0) {
2291         if(!((i_regs->isconst>>t)&1))
2292           emit_movimm(imm[i]<<16,t);
2293       }
2294     }
2295   }
2296   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2297     if(rt1[i]) {
2298       signed char s,t;
2299       t=get_reg(i_regs->regmap,rt1[i]);
2300       s=get_reg(i_regs->regmap,rs1[i]);
2301       if(rs1[i]) {
2302         //assert(t>=0);
2303         //assert(s>=0);
2304         if(t>=0) {
2305           if(!((i_regs->isconst>>t)&1)) {
2306             if(s<0) {
2307               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2308               emit_addimm(t,imm[i],t);
2309             }else{
2310               if(!((i_regs->wasconst>>s)&1))
2311                 emit_addimm(s,imm[i],t);
2312               else
2313                 emit_movimm(constmap[i][s]+imm[i],t);
2314             }
2315           }
2316         }
2317       } else {
2318         if(t>=0) {
2319           if(!((i_regs->isconst>>t)&1))
2320             emit_movimm(imm[i],t);
2321         }
2322       }
2323     }
2324   }
2325   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2326     if(rt1[i]) {
2327       signed char sl,tl;
2328       tl=get_reg(i_regs->regmap,rt1[i]);
2329       sl=get_reg(i_regs->regmap,rs1[i]);
2330       if(tl>=0) {
2331         if(rs1[i]) {
2332           assert(sl>=0);
2333           emit_addimm(sl,imm[i],tl);
2334         } else {
2335           emit_movimm(imm[i],tl);
2336         }
2337       }
2338     }
2339   }
2340   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2341     if(rt1[i]) {
2342       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2343       signed char sl,t;
2344       t=get_reg(i_regs->regmap,rt1[i]);
2345       sl=get_reg(i_regs->regmap,rs1[i]);
2346       //assert(t>=0);
2347       if(t>=0) {
2348         if(rs1[i]>0) {
2349             if(opcode[i]==0x0a) { // SLTI
2350               if(sl<0) {
2351                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2352                 emit_slti32(t,imm[i],t);
2353               }else{
2354                 emit_slti32(sl,imm[i],t);
2355               }
2356             }
2357             else { // SLTIU
2358               if(sl<0) {
2359                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2360                 emit_sltiu32(t,imm[i],t);
2361               }else{
2362                 emit_sltiu32(sl,imm[i],t);
2363               }
2364             }
2365         }else{
2366           // SLTI(U) with r0 is just stupid,
2367           // nonetheless examples can be found
2368           if(opcode[i]==0x0a) // SLTI
2369             if(0<imm[i]) emit_movimm(1,t);
2370             else emit_zeroreg(t);
2371           else // SLTIU
2372           {
2373             if(imm[i]) emit_movimm(1,t);
2374             else emit_zeroreg(t);
2375           }
2376         }
2377       }
2378     }
2379   }
2380   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2381     if(rt1[i]) {
2382       signed char sl,tl;
2383       tl=get_reg(i_regs->regmap,rt1[i]);
2384       sl=get_reg(i_regs->regmap,rs1[i]);
2385       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2386         if(opcode[i]==0x0c) //ANDI
2387         {
2388           if(rs1[i]) {
2389             if(sl<0) {
2390               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2391               emit_andimm(tl,imm[i],tl);
2392             }else{
2393               if(!((i_regs->wasconst>>sl)&1))
2394                 emit_andimm(sl,imm[i],tl);
2395               else
2396                 emit_movimm(constmap[i][sl]&imm[i],tl);
2397             }
2398           }
2399           else
2400             emit_zeroreg(tl);
2401         }
2402         else
2403         {
2404           if(rs1[i]) {
2405             if(sl<0) {
2406               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2407             }
2408             if(opcode[i]==0x0d) { // ORI
2409               if(sl<0) {
2410                 emit_orimm(tl,imm[i],tl);
2411               }else{
2412                 if(!((i_regs->wasconst>>sl)&1))
2413                   emit_orimm(sl,imm[i],tl);
2414                 else
2415                   emit_movimm(constmap[i][sl]|imm[i],tl);
2416               }
2417             }
2418             if(opcode[i]==0x0e) { // XORI
2419               if(sl<0) {
2420                 emit_xorimm(tl,imm[i],tl);
2421               }else{
2422                 if(!((i_regs->wasconst>>sl)&1))
2423                   emit_xorimm(sl,imm[i],tl);
2424                 else
2425                   emit_movimm(constmap[i][sl]^imm[i],tl);
2426               }
2427             }
2428           }
2429           else {
2430             emit_movimm(imm[i],tl);
2431           }
2432         }
2433       }
2434     }
2435   }
2436 }
2437
2438 void shiftimm_assemble(int i,struct regstat *i_regs)
2439 {
2440   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2441   {
2442     if(rt1[i]) {
2443       signed char s,t;
2444       t=get_reg(i_regs->regmap,rt1[i]);
2445       s=get_reg(i_regs->regmap,rs1[i]);
2446       //assert(t>=0);
2447       if(t>=0&&!((i_regs->isconst>>t)&1)){
2448         if(rs1[i]==0)
2449         {
2450           emit_zeroreg(t);
2451         }
2452         else
2453         {
2454           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2455           if(imm[i]) {
2456             if(opcode2[i]==0) // SLL
2457             {
2458               emit_shlimm(s<0?t:s,imm[i],t);
2459             }
2460             if(opcode2[i]==2) // SRL
2461             {
2462               emit_shrimm(s<0?t:s,imm[i],t);
2463             }
2464             if(opcode2[i]==3) // SRA
2465             {
2466               emit_sarimm(s<0?t:s,imm[i],t);
2467             }
2468           }else{
2469             // Shift by zero
2470             if(s>=0 && s!=t) emit_mov(s,t);
2471           }
2472         }
2473       }
2474       //emit_storereg(rt1[i],t); //DEBUG
2475     }
2476   }
2477   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2478   {
2479     assert(0);
2480   }
2481   if(opcode2[i]==0x3c) // DSLL32
2482   {
2483     assert(0);
2484   }
2485   if(opcode2[i]==0x3e) // DSRL32
2486   {
2487     assert(0);
2488   }
2489   if(opcode2[i]==0x3f) // DSRA32
2490   {
2491     assert(0);
2492   }
2493 }
2494
2495 #ifndef shift_assemble
2496 static void shift_assemble(int i,struct regstat *i_regs)
2497 {
2498   signed char s,t,shift;
2499   if (rt1[i] == 0)
2500     return;
2501   assert(opcode2[i]<=0x07); // SLLV/SRLV/SRAV
2502   t = get_reg(i_regs->regmap, rt1[i]);
2503   s = get_reg(i_regs->regmap, rs1[i]);
2504   shift = get_reg(i_regs->regmap, rs2[i]);
2505   if (t < 0)
2506     return;
2507
2508   if(rs1[i]==0)
2509     emit_zeroreg(t);
2510   else if(rs2[i]==0) {
2511     assert(s>=0);
2512     if(s!=t) emit_mov(s,t);
2513   }
2514   else {
2515     host_tempreg_acquire();
2516     emit_andimm(shift,31,HOST_TEMPREG);
2517     switch(opcode2[i]) {
2518     case 4: // SLLV
2519       emit_shl(s,HOST_TEMPREG,t);
2520       break;
2521     case 6: // SRLV
2522       emit_shr(s,HOST_TEMPREG,t);
2523       break;
2524     case 7: // SRAV
2525       emit_sar(s,HOST_TEMPREG,t);
2526       break;
2527     default:
2528       assert(0);
2529     }
2530     host_tempreg_release();
2531   }
2532 }
2533
2534 #endif
2535
2536 enum {
2537   MTYPE_8000 = 0,
2538   MTYPE_8020,
2539   MTYPE_0000,
2540   MTYPE_A000,
2541   MTYPE_1F80,
2542 };
2543
2544 static int get_ptr_mem_type(u_int a)
2545 {
2546   if(a < 0x00200000) {
2547     if(a<0x1000&&((start>>20)==0xbfc||(start>>24)==0xa0))
2548       // return wrong, must use memhandler for BIOS self-test to pass
2549       // 007 does similar stuff from a00 mirror, weird stuff
2550       return MTYPE_8000;
2551     return MTYPE_0000;
2552   }
2553   if(0x1f800000 <= a && a < 0x1f801000)
2554     return MTYPE_1F80;
2555   if(0x80200000 <= a && a < 0x80800000)
2556     return MTYPE_8020;
2557   if(0xa0000000 <= a && a < 0xa0200000)
2558     return MTYPE_A000;
2559   return MTYPE_8000;
2560 }
2561
2562 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override)
2563 {
2564   void *jaddr = NULL;
2565   int type=0;
2566   int mr=rs1[i];
2567   if(((smrv_strong|smrv_weak)>>mr)&1) {
2568     type=get_ptr_mem_type(smrv[mr]);
2569     //printf("set %08x @%08x r%d %d\n", smrv[mr], start+i*4, mr, type);
2570   }
2571   else {
2572     // use the mirror we are running on
2573     type=get_ptr_mem_type(start);
2574     //printf("set nospec   @%08x r%d %d\n", start+i*4, mr, type);
2575   }
2576
2577   if(type==MTYPE_8020) { // RAM 80200000+ mirror
2578     host_tempreg_acquire();
2579     emit_andimm(addr,~0x00e00000,HOST_TEMPREG);
2580     addr=*addr_reg_override=HOST_TEMPREG;
2581     type=0;
2582   }
2583   else if(type==MTYPE_0000) { // RAM 0 mirror
2584     host_tempreg_acquire();
2585     emit_orimm(addr,0x80000000,HOST_TEMPREG);
2586     addr=*addr_reg_override=HOST_TEMPREG;
2587     type=0;
2588   }
2589   else if(type==MTYPE_A000) { // RAM A mirror
2590     host_tempreg_acquire();
2591     emit_andimm(addr,~0x20000000,HOST_TEMPREG);
2592     addr=*addr_reg_override=HOST_TEMPREG;
2593     type=0;
2594   }
2595   else if(type==MTYPE_1F80) { // scratchpad
2596     if (psxH == (void *)0x1f800000) {
2597       host_tempreg_acquire();
2598       emit_xorimm(addr,0x1f800000,HOST_TEMPREG);
2599       emit_cmpimm(HOST_TEMPREG,0x1000);
2600       host_tempreg_release();
2601       jaddr=out;
2602       emit_jc(0);
2603     }
2604     else {
2605       // do the usual RAM check, jump will go to the right handler
2606       type=0;
2607     }
2608   }
2609
2610   if(type==0)
2611   {
2612     emit_cmpimm(addr,RAM_SIZE);
2613     jaddr=out;
2614     #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2615     // Hint to branch predictor that the branch is unlikely to be taken
2616     if(rs1[i]>=28)
2617       emit_jno_unlikely(0);
2618     else
2619     #endif
2620       emit_jno(0);
2621     if(ram_offset!=0) {
2622       host_tempreg_acquire();
2623       emit_addimm(addr,ram_offset,HOST_TEMPREG);
2624       addr=*addr_reg_override=HOST_TEMPREG;
2625     }
2626   }
2627
2628   return jaddr;
2629 }
2630
2631 // return memhandler, or get directly accessable address and return 0
2632 static void *get_direct_memhandler(void *table, u_int addr,
2633   enum stub_type type, uintptr_t *addr_host)
2634 {
2635   uintptr_t l1, l2 = 0;
2636   l1 = ((uintptr_t *)table)[addr>>12];
2637   if ((l1 & (1ul << (sizeof(l1)*8-1))) == 0) {
2638     uintptr_t v = l1 << 1;
2639     *addr_host = v + addr;
2640     return NULL;
2641   }
2642   else {
2643     l1 <<= 1;
2644     if (type == LOADB_STUB || type == LOADBU_STUB || type == STOREB_STUB)
2645       l2 = ((uintptr_t *)l1)[0x1000/4 + 0x1000/2 + (addr&0xfff)];
2646     else if (type == LOADH_STUB || type == LOADHU_STUB || type == STOREH_STUB)
2647       l2=((uintptr_t *)l1)[0x1000/4 + (addr&0xfff)/2];
2648     else
2649       l2=((uintptr_t *)l1)[(addr&0xfff)/4];
2650     if ((l2 & (1<<31)) == 0) {
2651       uintptr_t v = l2 << 1;
2652       *addr_host = v + (addr&0xfff);
2653       return NULL;
2654     }
2655     return (void *)(l2 << 1);
2656   }
2657 }
2658
2659 static u_int get_host_reglist(const signed char *regmap)
2660 {
2661   u_int reglist = 0, hr;
2662   for (hr = 0; hr < HOST_REGS; hr++) {
2663     if (hr != EXCLUDE_REG && regmap[hr] >= 0)
2664       reglist |= 1 << hr;
2665   }
2666   return reglist;
2667 }
2668
2669 static u_int reglist_exclude(u_int reglist, int r1, int r2)
2670 {
2671   if (r1 >= 0)
2672     reglist &= ~(1u << r1);
2673   if (r2 >= 0)
2674     reglist &= ~(1u << r2);
2675   return reglist;
2676 }
2677
2678 static void load_assemble(int i, const struct regstat *i_regs)
2679 {
2680   int s,tl,addr;
2681   int offset;
2682   void *jaddr=0;
2683   int memtarget=0,c=0;
2684   int fastio_reg_override=-1;
2685   u_int reglist=get_host_reglist(i_regs->regmap);
2686   tl=get_reg(i_regs->regmap,rt1[i]);
2687   s=get_reg(i_regs->regmap,rs1[i]);
2688   offset=imm[i];
2689   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2690   if(s>=0) {
2691     c=(i_regs->wasconst>>s)&1;
2692     if (c) {
2693       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2694     }
2695   }
2696   //printf("load_assemble: c=%d\n",c);
2697   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2698   // FIXME: Even if the load is a NOP, we should check for pagefaults...
2699   if((tl<0&&(!c||(((u_int)constmap[i][s]+offset)>>16)==0x1f80))
2700     ||rt1[i]==0) {
2701       // could be FIFO, must perform the read
2702       // ||dummy read
2703       assem_debug("(forced read)\n");
2704       tl=get_reg(i_regs->regmap,-1);
2705       assert(tl>=0);
2706   }
2707   if(offset||s<0||c) addr=tl;
2708   else addr=s;
2709   //if(tl<0) tl=get_reg(i_regs->regmap,-1);
2710  if(tl>=0) {
2711   //printf("load_assemble: c=%d\n",c);
2712   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2713   assert(tl>=0); // Even if the load is a NOP, we must check for pagefaults and I/O
2714   reglist&=~(1<<tl);
2715   if(!c) {
2716     #ifdef R29_HACK
2717     // Strmnnrmn's speed hack
2718     if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
2719     #endif
2720     {
2721       jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override);
2722     }
2723   }
2724   else if(ram_offset&&memtarget) {
2725     host_tempreg_acquire();
2726     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2727     fastio_reg_override=HOST_TEMPREG;
2728   }
2729   int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg
2730   if (opcode[i]==0x20) { // LB
2731     if(!c||memtarget) {
2732       if(!dummy) {
2733         {
2734           int x=0,a=tl;
2735           if(!c) a=addr;
2736           if(fastio_reg_override>=0) a=fastio_reg_override;
2737
2738           emit_movsbl_indexed(x,a,tl);
2739         }
2740       }
2741       if(jaddr)
2742         add_stub_r(LOADB_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2743     }
2744     else
2745       inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2746   }
2747   if (opcode[i]==0x21) { // LH
2748     if(!c||memtarget) {
2749       if(!dummy) {
2750         int x=0,a=tl;
2751         if(!c) a=addr;
2752         if(fastio_reg_override>=0) a=fastio_reg_override;
2753         emit_movswl_indexed(x,a,tl);
2754       }
2755       if(jaddr)
2756         add_stub_r(LOADH_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2757     }
2758     else
2759       inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2760   }
2761   if (opcode[i]==0x23) { // LW
2762     if(!c||memtarget) {
2763       if(!dummy) {
2764         int a=addr;
2765         if(fastio_reg_override>=0) a=fastio_reg_override;
2766         emit_readword_indexed(0,a,tl);
2767       }
2768       if(jaddr)
2769         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2770     }
2771     else
2772       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2773   }
2774   if (opcode[i]==0x24) { // LBU
2775     if(!c||memtarget) {
2776       if(!dummy) {
2777         int x=0,a=tl;
2778         if(!c) a=addr;
2779         if(fastio_reg_override>=0) a=fastio_reg_override;
2780
2781         emit_movzbl_indexed(x,a,tl);
2782       }
2783       if(jaddr)
2784         add_stub_r(LOADBU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2785     }
2786     else
2787       inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2788   }
2789   if (opcode[i]==0x25) { // LHU
2790     if(!c||memtarget) {
2791       if(!dummy) {
2792         int x=0,a=tl;
2793         if(!c) a=addr;
2794         if(fastio_reg_override>=0) a=fastio_reg_override;
2795         emit_movzwl_indexed(x,a,tl);
2796       }
2797       if(jaddr)
2798         add_stub_r(LOADHU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2799     }
2800     else
2801       inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2802   }
2803   if (opcode[i]==0x27) { // LWU
2804     assert(0);
2805   }
2806   if (opcode[i]==0x37) { // LD
2807     assert(0);
2808   }
2809  }
2810  if (fastio_reg_override == HOST_TEMPREG)
2811    host_tempreg_release();
2812 }
2813
2814 #ifndef loadlr_assemble
2815 static void loadlr_assemble(int i, const struct regstat *i_regs)
2816 {
2817   int s,tl,temp,temp2,addr;
2818   int offset;
2819   void *jaddr=0;
2820   int memtarget=0,c=0;
2821   int fastio_reg_override=-1;
2822   u_int reglist=get_host_reglist(i_regs->regmap);
2823   tl=get_reg(i_regs->regmap,rt1[i]);
2824   s=get_reg(i_regs->regmap,rs1[i]);
2825   temp=get_reg(i_regs->regmap,-1);
2826   temp2=get_reg(i_regs->regmap,FTEMP);
2827   addr=get_reg(i_regs->regmap,AGEN1+(i&1));
2828   assert(addr<0);
2829   offset=imm[i];
2830   reglist|=1<<temp;
2831   if(offset||s<0||c) addr=temp2;
2832   else addr=s;
2833   if(s>=0) {
2834     c=(i_regs->wasconst>>s)&1;
2835     if(c) {
2836       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2837     }
2838   }
2839   if(!c) {
2840     emit_shlimm(addr,3,temp);
2841     if (opcode[i]==0x22||opcode[i]==0x26) {
2842       emit_andimm(addr,0xFFFFFFFC,temp2); // LWL/LWR
2843     }else{
2844       emit_andimm(addr,0xFFFFFFF8,temp2); // LDL/LDR
2845     }
2846     jaddr=emit_fastpath_cmp_jump(i,temp2,&fastio_reg_override);
2847   }
2848   else {
2849     if(ram_offset&&memtarget) {
2850       host_tempreg_acquire();
2851       emit_addimm(temp2,ram_offset,HOST_TEMPREG);
2852       fastio_reg_override=HOST_TEMPREG;
2853     }
2854     if (opcode[i]==0x22||opcode[i]==0x26) {
2855       emit_movimm(((constmap[i][s]+offset)<<3)&24,temp); // LWL/LWR
2856     }else{
2857       emit_movimm(((constmap[i][s]+offset)<<3)&56,temp); // LDL/LDR
2858     }
2859   }
2860   if (opcode[i]==0x22||opcode[i]==0x26) { // LWL/LWR
2861     if(!c||memtarget) {
2862       int a=temp2;
2863       if(fastio_reg_override>=0) a=fastio_reg_override;
2864       emit_readword_indexed(0,a,temp2);
2865       if(fastio_reg_override==HOST_TEMPREG) host_tempreg_release();
2866       if(jaddr) add_stub_r(LOADW_STUB,jaddr,out,i,temp2,i_regs,ccadj[i],reglist);
2867     }
2868     else
2869       inline_readstub(LOADW_STUB,i,(constmap[i][s]+offset)&0xFFFFFFFC,i_regs->regmap,FTEMP,ccadj[i],reglist);
2870     if(rt1[i]) {
2871       assert(tl>=0);
2872       emit_andimm(temp,24,temp);
2873       if (opcode[i]==0x22) // LWL
2874         emit_xorimm(temp,24,temp);
2875       host_tempreg_acquire();
2876       emit_movimm(-1,HOST_TEMPREG);
2877       if (opcode[i]==0x26) {
2878         emit_shr(temp2,temp,temp2);
2879         emit_bic_lsr(tl,HOST_TEMPREG,temp,tl);
2880       }else{
2881         emit_shl(temp2,temp,temp2);
2882         emit_bic_lsl(tl,HOST_TEMPREG,temp,tl);
2883       }
2884       host_tempreg_release();
2885       emit_or(temp2,tl,tl);
2886     }
2887     //emit_storereg(rt1[i],tl); // DEBUG
2888   }
2889   if (opcode[i]==0x1A||opcode[i]==0x1B) { // LDL/LDR
2890     assert(0);
2891   }
2892 }
2893 #endif
2894
2895 void store_assemble(int i, const struct regstat *i_regs)
2896 {
2897   int s,tl;
2898   int addr,temp;
2899   int offset;
2900   void *jaddr=0;
2901   enum stub_type type;
2902   int memtarget=0,c=0;
2903   int agr=AGEN1+(i&1);
2904   int fastio_reg_override=-1;
2905   u_int reglist=get_host_reglist(i_regs->regmap);
2906   tl=get_reg(i_regs->regmap,rs2[i]);
2907   s=get_reg(i_regs->regmap,rs1[i]);
2908   temp=get_reg(i_regs->regmap,agr);
2909   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2910   offset=imm[i];
2911   if(s>=0) {
2912     c=(i_regs->wasconst>>s)&1;
2913     if(c) {
2914       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2915     }
2916   }
2917   assert(tl>=0);
2918   assert(temp>=0);
2919   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2920   if(offset||s<0||c) addr=temp;
2921   else addr=s;
2922   if(!c) {
2923     jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override);
2924   }
2925   else if(ram_offset&&memtarget) {
2926     host_tempreg_acquire();
2927     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2928     fastio_reg_override=HOST_TEMPREG;
2929   }
2930
2931   if (opcode[i]==0x28) { // SB
2932     if(!c||memtarget) {
2933       int x=0,a=temp;
2934       if(!c) a=addr;
2935       if(fastio_reg_override>=0) a=fastio_reg_override;
2936       emit_writebyte_indexed(tl,x,a);
2937     }
2938     type=STOREB_STUB;
2939   }
2940   if (opcode[i]==0x29) { // SH
2941     if(!c||memtarget) {
2942       int x=0,a=temp;
2943       if(!c) a=addr;
2944       if(fastio_reg_override>=0) a=fastio_reg_override;
2945       emit_writehword_indexed(tl,x,a);
2946     }
2947     type=STOREH_STUB;
2948   }
2949   if (opcode[i]==0x2B) { // SW
2950     if(!c||memtarget) {
2951       int a=addr;
2952       if(fastio_reg_override>=0) a=fastio_reg_override;
2953       emit_writeword_indexed(tl,0,a);
2954     }
2955     type=STOREW_STUB;
2956   }
2957   if (opcode[i]==0x3F) { // SD
2958     assert(0);
2959     type=STORED_STUB;
2960   }
2961   if(fastio_reg_override==HOST_TEMPREG)
2962     host_tempreg_release();
2963   if(jaddr) {
2964     // PCSX store handlers don't check invcode again
2965     reglist|=1<<addr;
2966     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2967     jaddr=0;
2968   }
2969   if(!(i_regs->waswritten&(1<<rs1[i])) && !HACK_ENABLED(NDHACK_NO_SMC_CHECK)) {
2970     if(!c||memtarget) {
2971       #ifdef DESTRUCTIVE_SHIFT
2972       // The x86 shift operation is 'destructive'; it overwrites the
2973       // source register, so we need to make a copy first and use that.
2974       addr=temp;
2975       #endif
2976       #if defined(HOST_IMM8)
2977       int ir=get_reg(i_regs->regmap,INVCP);
2978       assert(ir>=0);
2979       emit_cmpmem_indexedsr12_reg(ir,addr,1);
2980       #else
2981       emit_cmpmem_indexedsr12_imm(invalid_code,addr,1);
2982       #endif
2983       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2984       emit_callne(invalidate_addr_reg[addr]);
2985       #else
2986       void *jaddr2 = out;
2987       emit_jne(0);
2988       add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),addr,0,0,0);
2989       #endif
2990     }
2991   }
2992   u_int addr_val=constmap[i][s]+offset;
2993   if(jaddr) {
2994     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2995   } else if(c&&!memtarget) {
2996     inline_writestub(type,i,addr_val,i_regs->regmap,rs2[i],ccadj[i],reglist);
2997   }
2998   // basic current block modification detection..
2999   // not looking back as that should be in mips cache already
3000   // (see Spyro2 title->attract mode)
3001   if(c&&start+i*4<addr_val&&addr_val<start+slen*4) {
3002     SysPrintf("write to %08x hits block %08x, pc=%08x\n",addr_val,start,start+i*4);
3003     assert(i_regs->regmap==regs[i].regmap); // not delay slot
3004     if(i_regs->regmap==regs[i].regmap) {
3005       load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
3006       wb_dirtys(regs[i].regmap_entry,regs[i].wasdirty);
3007       emit_movimm(start+i*4+4,0);
3008       emit_writeword(0,&pcaddr);
3009       emit_addimm(HOST_CCREG,2,HOST_CCREG);
3010       emit_far_call(get_addr_ht);
3011       emit_jmpreg(0);
3012     }
3013   }
3014 }
3015
3016 static void storelr_assemble(int i, const struct regstat *i_regs)
3017 {
3018   int s,tl;
3019   int temp;
3020   int offset;
3021   void *jaddr=0;
3022   void *case1, *case2, *case3;
3023   void *done0, *done1, *done2;
3024   int memtarget=0,c=0;
3025   int agr=AGEN1+(i&1);
3026   u_int reglist=get_host_reglist(i_regs->regmap);
3027   tl=get_reg(i_regs->regmap,rs2[i]);
3028   s=get_reg(i_regs->regmap,rs1[i]);
3029   temp=get_reg(i_regs->regmap,agr);
3030   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3031   offset=imm[i];
3032   if(s>=0) {
3033     c=(i_regs->isconst>>s)&1;
3034     if(c) {
3035       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
3036     }
3037   }
3038   assert(tl>=0);
3039   assert(temp>=0);
3040   if(!c) {
3041     emit_cmpimm(s<0||offset?temp:s,RAM_SIZE);
3042     if(!offset&&s!=temp) emit_mov(s,temp);
3043     jaddr=out;
3044     emit_jno(0);
3045   }
3046   else
3047   {
3048     if(!memtarget||!rs1[i]) {
3049       jaddr=out;
3050       emit_jmp(0);
3051     }
3052   }
3053   if(ram_offset)
3054     emit_addimm_no_flags(ram_offset,temp);
3055
3056   if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
3057     assert(0);
3058   }
3059
3060   emit_xorimm(temp,3,temp);
3061   emit_testimm(temp,2);
3062   case2=out;
3063   emit_jne(0);
3064   emit_testimm(temp,1);
3065   case1=out;
3066   emit_jne(0);
3067   // 0
3068   if (opcode[i]==0x2A) { // SWL
3069     emit_writeword_indexed(tl,0,temp);
3070   }
3071   else if (opcode[i]==0x2E) { // SWR
3072     emit_writebyte_indexed(tl,3,temp);
3073   }
3074   else
3075     assert(0);
3076   done0=out;
3077   emit_jmp(0);
3078   // 1
3079   set_jump_target(case1, out);
3080   if (opcode[i]==0x2A) { // SWL
3081     // Write 3 msb into three least significant bytes
3082     if(rs2[i]) emit_rorimm(tl,8,tl);
3083     emit_writehword_indexed(tl,-1,temp);
3084     if(rs2[i]) emit_rorimm(tl,16,tl);
3085     emit_writebyte_indexed(tl,1,temp);
3086     if(rs2[i]) emit_rorimm(tl,8,tl);
3087   }
3088   else if (opcode[i]==0x2E) { // SWR
3089     // Write two lsb into two most significant bytes
3090     emit_writehword_indexed(tl,1,temp);
3091   }
3092   done1=out;
3093   emit_jmp(0);
3094   // 2
3095   set_jump_target(case2, out);
3096   emit_testimm(temp,1);
3097   case3=out;
3098   emit_jne(0);
3099   if (opcode[i]==0x2A) { // SWL
3100     // Write two msb into two least significant bytes
3101     if(rs2[i]) emit_rorimm(tl,16,tl);
3102     emit_writehword_indexed(tl,-2,temp);
3103     if(rs2[i]) emit_rorimm(tl,16,tl);
3104   }
3105   else if (opcode[i]==0x2E) { // SWR
3106     // Write 3 lsb into three most significant bytes
3107     emit_writebyte_indexed(tl,-1,temp);
3108     if(rs2[i]) emit_rorimm(tl,8,tl);
3109     emit_writehword_indexed(tl,0,temp);
3110     if(rs2[i]) emit_rorimm(tl,24,tl);
3111   }
3112   done2=out;
3113   emit_jmp(0);
3114   // 3
3115   set_jump_target(case3, out);
3116   if (opcode[i]==0x2A) { // SWL
3117     // Write msb into least significant byte
3118     if(rs2[i]) emit_rorimm(tl,24,tl);
3119     emit_writebyte_indexed(tl,-3,temp);
3120     if(rs2[i]) emit_rorimm(tl,8,tl);
3121   }
3122   else if (opcode[i]==0x2E) { // SWR
3123     // Write entire word
3124     emit_writeword_indexed(tl,-3,temp);
3125   }
3126   set_jump_target(done0, out);
3127   set_jump_target(done1, out);
3128   set_jump_target(done2, out);
3129   if(!c||!memtarget)
3130     add_stub_r(STORELR_STUB,jaddr,out,i,temp,i_regs,ccadj[i],reglist);
3131   if(!(i_regs->waswritten&(1<<rs1[i])) && !HACK_ENABLED(NDHACK_NO_SMC_CHECK)) {
3132     emit_addimm_no_flags(-ram_offset,temp);
3133     #if defined(HOST_IMM8)
3134     int ir=get_reg(i_regs->regmap,INVCP);
3135     assert(ir>=0);
3136     emit_cmpmem_indexedsr12_reg(ir,temp,1);
3137     #else
3138     emit_cmpmem_indexedsr12_imm(invalid_code,temp,1);
3139     #endif
3140     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3141     emit_callne(invalidate_addr_reg[temp]);
3142     #else
3143     void *jaddr2 = out;
3144     emit_jne(0);
3145     add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),temp,0,0,0);
3146     #endif
3147   }
3148 }
3149
3150 static void cop0_assemble(int i,struct regstat *i_regs)
3151 {
3152   if(opcode2[i]==0) // MFC0
3153   {
3154     signed char t=get_reg(i_regs->regmap,rt1[i]);
3155     u_int copr=(source[i]>>11)&0x1f;
3156     //assert(t>=0); // Why does this happen?  OOT is weird
3157     if(t>=0&&rt1[i]!=0) {
3158       emit_readword(&reg_cop0[copr],t);
3159     }
3160   }
3161   else if(opcode2[i]==4) // MTC0
3162   {
3163     signed char s=get_reg(i_regs->regmap,rs1[i]);
3164     char copr=(source[i]>>11)&0x1f;
3165     assert(s>=0);
3166     wb_register(rs1[i],i_regs->regmap,i_regs->dirty);
3167     if(copr==9||copr==11||copr==12||copr==13) {
3168       emit_readword(&last_count,HOST_TEMPREG);
3169       emit_loadreg(CCREG,HOST_CCREG); // TODO: do proper reg alloc
3170       emit_add(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
3171       emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3172       emit_writeword(HOST_CCREG,&Count);
3173     }
3174     // What a mess.  The status register (12) can enable interrupts,
3175     // so needs a special case to handle a pending interrupt.
3176     // The interrupt must be taken immediately, because a subsequent
3177     // instruction might disable interrupts again.
3178     if(copr==12||copr==13) {
3179       if (is_delayslot) {
3180         // burn cycles to cause cc_interrupt, which will
3181         // reschedule next_interupt. Relies on CCREG from above.
3182         assem_debug("MTC0 DS %d\n", copr);
3183         emit_writeword(HOST_CCREG,&last_count);
3184         emit_movimm(0,HOST_CCREG);
3185         emit_storereg(CCREG,HOST_CCREG);
3186         emit_loadreg(rs1[i],1);
3187         emit_movimm(copr,0);
3188         emit_far_call(pcsx_mtc0_ds);
3189         emit_loadreg(rs1[i],s);
3190         return;
3191       }
3192       emit_movimm(start+i*4+4,HOST_TEMPREG);
3193       emit_writeword(HOST_TEMPREG,&pcaddr);
3194       emit_movimm(0,HOST_TEMPREG);
3195       emit_writeword(HOST_TEMPREG,&pending_exception);
3196     }
3197     if(s==HOST_CCREG)
3198       emit_loadreg(rs1[i],1);
3199     else if(s!=1)
3200       emit_mov(s,1);
3201     emit_movimm(copr,0);
3202     emit_far_call(pcsx_mtc0);
3203     if(copr==9||copr==11||copr==12||copr==13) {
3204       emit_readword(&Count,HOST_CCREG);
3205       emit_readword(&next_interupt,HOST_TEMPREG);
3206       emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3207       emit_sub(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
3208       emit_writeword(HOST_TEMPREG,&last_count);
3209       emit_storereg(CCREG,HOST_CCREG);
3210     }
3211     if(copr==12||copr==13) {
3212       assert(!is_delayslot);
3213       emit_readword(&pending_exception,14);
3214       emit_test(14,14);
3215       void *jaddr = out;
3216       emit_jeq(0);
3217       emit_readword(&pcaddr, 0);
3218       emit_addimm(HOST_CCREG,2,HOST_CCREG);
3219       emit_far_call(get_addr_ht);
3220       emit_jmpreg(0);
3221       set_jump_target(jaddr, out);
3222     }
3223     emit_loadreg(rs1[i],s);
3224   }
3225   else
3226   {
3227     assert(opcode2[i]==0x10);
3228     //if((source[i]&0x3f)==0x10) // RFE
3229     {
3230       emit_readword(&Status,0);
3231       emit_andimm(0,0x3c,1);
3232       emit_andimm(0,~0xf,0);
3233       emit_orrshr_imm(1,2,0);
3234       emit_writeword(0,&Status);
3235     }
3236   }
3237 }
3238
3239 static void cop1_unusable(int i,struct regstat *i_regs)
3240 {
3241   // XXX: should just just do the exception instead
3242   //if(!cop1_usable)
3243   {
3244     void *jaddr=out;
3245     emit_jmp(0);
3246     add_stub_r(FP_STUB,jaddr,out,i,0,i_regs,is_delayslot,0);
3247   }
3248 }
3249
3250 static void cop1_assemble(int i,struct regstat *i_regs)
3251 {
3252   cop1_unusable(i, i_regs);
3253 }
3254
3255 static void c1ls_assemble(int i,struct regstat *i_regs)
3256 {
3257   cop1_unusable(i, i_regs);
3258 }
3259
3260 // FP_STUB
3261 static void do_cop1stub(int n)
3262 {
3263   literal_pool(256);
3264   assem_debug("do_cop1stub %x\n",start+stubs[n].a*4);
3265   set_jump_target(stubs[n].addr, out);
3266   int i=stubs[n].a;
3267 //  int rs=stubs[n].b;
3268   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3269   int ds=stubs[n].d;
3270   if(!ds) {
3271     load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
3272     //if(i_regs!=&regs[i]) printf("oops: regs[i]=%x i_regs=%x",(int)&regs[i],(int)i_regs);
3273   }
3274   //else {printf("fp exception in delay slot\n");}
3275   wb_dirtys(i_regs->regmap_entry,i_regs->wasdirty);
3276   if(regs[i].regmap_entry[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
3277   emit_movimm(start+(i-ds)*4,EAX); // Get PC
3278   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3279   emit_far_jump(ds?fp_exception_ds:fp_exception);
3280 }
3281
3282 // assumes callee-save regs are already saved
3283 static void cop2_call_stall_check(u_int op, int i, const struct regstat *i_regs, u_int reglist)
3284 {
3285   if (HACK_ENABLED(NDHACK_GTE_NO_STALL))
3286     return;
3287   //assert(get_reg(i_regs->regmap, CCREG) == HOST_CCREG);
3288   if (get_reg(i_regs->regmap, CCREG) != HOST_CCREG) {
3289     // happens occasionally... cc evicted? Don't bother then
3290     //printf("no cc %08x\n", start + i*4);
3291     return;
3292   }
3293   assem_debug("cop2_call_stall_check\n");
3294   save_regs(reglist);
3295   emit_movimm(gte_cycletab[op], 0);
3296   emit_addimm(HOST_CCREG, CLOCK_ADJUST(ccadj[i]), 1);
3297   emit_far_call(call_gteStall);
3298   restore_regs(reglist);
3299 }
3300
3301 static void cop2_get_dreg(u_int copr,signed char tl,signed char temp)
3302 {
3303   switch (copr) {
3304     case 1:
3305     case 3:
3306     case 5:
3307     case 8:
3308     case 9:
3309     case 10:
3310     case 11:
3311       emit_readword(&reg_cop2d[copr],tl);
3312       emit_signextend16(tl,tl);
3313       emit_writeword(tl,&reg_cop2d[copr]); // hmh
3314       break;
3315     case 7:
3316     case 16:
3317     case 17:
3318     case 18:
3319     case 19:
3320       emit_readword(&reg_cop2d[copr],tl);
3321       emit_andimm(tl,0xffff,tl);
3322       emit_writeword(tl,&reg_cop2d[copr]);
3323       break;
3324     case 15:
3325       emit_readword(&reg_cop2d[14],tl); // SXY2
3326       emit_writeword(tl,&reg_cop2d[copr]);
3327       break;
3328     case 28:
3329     case 29:
3330       c2op_mfc2_29_assemble(tl,temp);
3331       break;
3332     default:
3333       emit_readword(&reg_cop2d[copr],tl);
3334       break;
3335   }
3336 }
3337
3338 static void cop2_put_dreg(u_int copr,signed char sl,signed char temp)
3339 {
3340   switch (copr) {
3341     case 15:
3342       emit_readword(&reg_cop2d[13],temp);  // SXY1
3343       emit_writeword(sl,&reg_cop2d[copr]);
3344       emit_writeword(temp,&reg_cop2d[12]); // SXY0
3345       emit_readword(&reg_cop2d[14],temp);  // SXY2
3346       emit_writeword(sl,&reg_cop2d[14]);
3347       emit_writeword(temp,&reg_cop2d[13]); // SXY1
3348       break;
3349     case 28:
3350       emit_andimm(sl,0x001f,temp);
3351       emit_shlimm(temp,7,temp);
3352       emit_writeword(temp,&reg_cop2d[9]);
3353       emit_andimm(sl,0x03e0,temp);
3354       emit_shlimm(temp,2,temp);
3355       emit_writeword(temp,&reg_cop2d[10]);
3356       emit_andimm(sl,0x7c00,temp);
3357       emit_shrimm(temp,3,temp);
3358       emit_writeword(temp,&reg_cop2d[11]);
3359       emit_writeword(sl,&reg_cop2d[28]);
3360       break;
3361     case 30:
3362       emit_xorsar_imm(sl,sl,31,temp);
3363 #if defined(HAVE_ARMV5) || defined(__aarch64__)
3364       emit_clz(temp,temp);
3365 #else
3366       emit_movs(temp,HOST_TEMPREG);
3367       emit_movimm(0,temp);
3368       emit_jeq((int)out+4*4);
3369       emit_addpl_imm(temp,1,temp);
3370       emit_lslpls_imm(HOST_TEMPREG,1,HOST_TEMPREG);
3371       emit_jns((int)out-2*4);
3372 #endif
3373       emit_writeword(sl,&reg_cop2d[30]);
3374       emit_writeword(temp,&reg_cop2d[31]);
3375       break;
3376     case 31:
3377       break;
3378     default:
3379       emit_writeword(sl,&reg_cop2d[copr]);
3380       break;
3381   }
3382 }
3383
3384 static void c2ls_assemble(int i, const struct regstat *i_regs)
3385 {
3386   int s,tl;
3387   int ar;
3388   int offset;
3389   int memtarget=0,c=0;
3390   void *jaddr2=NULL;
3391   enum stub_type type;
3392   int agr=AGEN1+(i&1);
3393   int fastio_reg_override=-1;
3394   u_int reglist=get_host_reglist(i_regs->regmap);
3395   u_int copr=(source[i]>>16)&0x1f;
3396   s=get_reg(i_regs->regmap,rs1[i]);
3397   tl=get_reg(i_regs->regmap,FTEMP);
3398   offset=imm[i];
3399   assert(rs1[i]>0);
3400   assert(tl>=0);
3401
3402   if(i_regs->regmap[HOST_CCREG]==CCREG)
3403     reglist&=~(1<<HOST_CCREG);
3404
3405   // get the address
3406   if (opcode[i]==0x3a) { // SWC2
3407     ar=get_reg(i_regs->regmap,agr);
3408     if(ar<0) ar=get_reg(i_regs->regmap,-1);
3409     reglist|=1<<ar;
3410   } else { // LWC2
3411     ar=tl;
3412   }
3413   if(s>=0) c=(i_regs->wasconst>>s)&1;
3414   memtarget=c&&(((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE);
3415   if (!offset&&!c&&s>=0) ar=s;
3416   assert(ar>=0);
3417
3418   if (opcode[i]==0x3a) { // SWC2
3419     cop2_call_stall_check(0, i, i_regs, reglist_exclude(reglist, tl, -1));
3420     cop2_get_dreg(copr,tl,-1);
3421     type=STOREW_STUB;
3422   }
3423   else
3424     type=LOADW_STUB;
3425
3426   if(c&&!memtarget) {
3427     jaddr2=out;
3428     emit_jmp(0); // inline_readstub/inline_writestub?
3429   }
3430   else {
3431     if(!c) {
3432       jaddr2=emit_fastpath_cmp_jump(i,ar,&fastio_reg_override);
3433     }
3434     else if(ram_offset&&memtarget) {
3435       host_tempreg_acquire();
3436       emit_addimm(ar,ram_offset,HOST_TEMPREG);
3437       fastio_reg_override=HOST_TEMPREG;
3438     }
3439     if (opcode[i]==0x32) { // LWC2
3440       int a=ar;
3441       if(fastio_reg_override>=0) a=fastio_reg_override;
3442       emit_readword_indexed(0,a,tl);
3443     }
3444     if (opcode[i]==0x3a) { // SWC2
3445       #ifdef DESTRUCTIVE_SHIFT
3446       if(!offset&&!c&&s>=0) emit_mov(s,ar);
3447       #endif
3448       int a=ar;
3449       if(fastio_reg_override>=0) a=fastio_reg_override;
3450       emit_writeword_indexed(tl,0,a);
3451     }
3452   }
3453   if(fastio_reg_override==HOST_TEMPREG)
3454     host_tempreg_release();
3455   if(jaddr2)
3456     add_stub_r(type,jaddr2,out,i,ar,i_regs,ccadj[i],reglist);
3457   if(opcode[i]==0x3a) // SWC2
3458   if(!(i_regs->waswritten&(1<<rs1[i])) && !HACK_ENABLED(NDHACK_NO_SMC_CHECK)) {
3459 #if defined(HOST_IMM8)
3460     int ir=get_reg(i_regs->regmap,INVCP);
3461     assert(ir>=0);
3462     emit_cmpmem_indexedsr12_reg(ir,ar,1);
3463 #else
3464     emit_cmpmem_indexedsr12_imm(invalid_code,ar,1);
3465 #endif
3466     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3467     emit_callne(invalidate_addr_reg[ar]);
3468     #else
3469     void *jaddr3 = out;
3470     emit_jne(0);
3471     add_stub(INVCODE_STUB,jaddr3,out,reglist|(1<<HOST_CCREG),ar,0,0,0);
3472     #endif
3473   }
3474   if (opcode[i]==0x32) { // LWC2
3475     host_tempreg_acquire();
3476     cop2_put_dreg(copr,tl,HOST_TEMPREG);
3477     host_tempreg_release();
3478   }
3479 }
3480
3481 static void cop2_assemble(int i, const struct regstat *i_regs)
3482 {
3483   u_int copr = (source[i]>>11) & 0x1f;
3484   signed char temp = get_reg(i_regs->regmap, -1);
3485
3486   if (opcode2[i] == 0 || opcode2[i] == 2) { // MFC2/CFC2
3487     if (!HACK_ENABLED(NDHACK_GTE_NO_STALL)) {
3488       signed char tl = get_reg(i_regs->regmap, rt1[i]);
3489       u_int reglist = reglist_exclude(get_host_reglist(i_regs->regmap), tl, temp);
3490       cop2_call_stall_check(0, i, i_regs, reglist);
3491     }
3492   }
3493   if (opcode2[i]==0) { // MFC2
3494     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3495     if(tl>=0&&rt1[i]!=0)
3496       cop2_get_dreg(copr,tl,temp);
3497   }
3498   else if (opcode2[i]==4) { // MTC2
3499     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3500     cop2_put_dreg(copr,sl,temp);
3501   }
3502   else if (opcode2[i]==2) // CFC2
3503   {
3504     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3505     if(tl>=0&&rt1[i]!=0)
3506       emit_readword(&reg_cop2c[copr],tl);
3507   }
3508   else if (opcode2[i]==6) // CTC2
3509   {
3510     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3511     switch(copr) {
3512       case 4:
3513       case 12:
3514       case 20:
3515       case 26:
3516       case 27:
3517       case 29:
3518       case 30:
3519         emit_signextend16(sl,temp);
3520         break;
3521       case 31:
3522         c2op_ctc2_31_assemble(sl,temp);
3523         break;
3524       default:
3525         temp=sl;
3526         break;
3527     }
3528     emit_writeword(temp,&reg_cop2c[copr]);
3529     assert(sl>=0);
3530   }
3531 }
3532
3533 static void do_unalignedwritestub(int n)
3534 {
3535   assem_debug("do_unalignedwritestub %x\n",start+stubs[n].a*4);
3536   literal_pool(256);
3537   set_jump_target(stubs[n].addr, out);
3538
3539   int i=stubs[n].a;
3540   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3541   int addr=stubs[n].b;
3542   u_int reglist=stubs[n].e;
3543   signed char *i_regmap=i_regs->regmap;
3544   int temp2=get_reg(i_regmap,FTEMP);
3545   int rt;
3546   rt=get_reg(i_regmap,rs2[i]);
3547   assert(rt>=0);
3548   assert(addr>=0);
3549   assert(opcode[i]==0x2a||opcode[i]==0x2e); // SWL/SWR only implemented
3550   reglist|=(1<<addr);
3551   reglist&=~(1<<temp2);
3552
3553 #if 1
3554   // don't bother with it and call write handler
3555   save_regs(reglist);
3556   pass_args(addr,rt);
3557   int cc=get_reg(i_regmap,CCREG);
3558   if(cc<0)
3559     emit_loadreg(CCREG,2);
3560   emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2);
3561   emit_far_call((opcode[i]==0x2a?jump_handle_swl:jump_handle_swr));
3562   emit_addimm(0,-CLOCK_ADJUST((int)stubs[n].d+1),cc<0?2:cc);
3563   if(cc<0)
3564     emit_storereg(CCREG,2);
3565   restore_regs(reglist);
3566   emit_jmp(stubs[n].retaddr); // return address
3567 #else
3568   emit_andimm(addr,0xfffffffc,temp2);
3569   emit_writeword(temp2,&address);
3570
3571   save_regs(reglist);
3572   emit_shrimm(addr,16,1);
3573   int cc=get_reg(i_regmap,CCREG);
3574   if(cc<0) {
3575     emit_loadreg(CCREG,2);
3576   }
3577   emit_movimm((u_int)readmem,0);
3578   emit_addimm(cc<0?2:cc,2*stubs[n].d+2,2);
3579   emit_call((int)&indirect_jump_indexed);
3580   restore_regs(reglist);
3581
3582   emit_readword(&readmem_dword,temp2);
3583   int temp=addr; //hmh
3584   emit_shlimm(addr,3,temp);
3585   emit_andimm(temp,24,temp);
3586   if (opcode[i]==0x2a) // SWL
3587     emit_xorimm(temp,24,temp);
3588   emit_movimm(-1,HOST_TEMPREG);
3589   if (opcode[i]==0x2a) { // SWL
3590     emit_bic_lsr(temp2,HOST_TEMPREG,temp,temp2);
3591     emit_orrshr(rt,temp,temp2);
3592   }else{
3593     emit_bic_lsl(temp2,HOST_TEMPREG,temp,temp2);
3594     emit_orrshl(rt,temp,temp2);
3595   }
3596   emit_readword(&address,addr);
3597   emit_writeword(temp2,&word);
3598   //save_regs(reglist); // don't need to, no state changes
3599   emit_shrimm(addr,16,1);
3600   emit_movimm((u_int)writemem,0);
3601   //emit_call((int)&indirect_jump_indexed);
3602   emit_mov(15,14);
3603   emit_readword_dualindexedx4(0,1,15);
3604   emit_readword(&Count,HOST_TEMPREG);
3605   emit_readword(&next_interupt,2);
3606   emit_addimm(HOST_TEMPREG,-2*stubs[n].d-2,HOST_TEMPREG);
3607   emit_writeword(2,&last_count);
3608   emit_sub(HOST_TEMPREG,2,cc<0?HOST_TEMPREG:cc);
3609   if(cc<0) {
3610     emit_storereg(CCREG,HOST_TEMPREG);
3611   }
3612   restore_regs(reglist);
3613   emit_jmp(stubs[n].retaddr); // return address
3614 #endif
3615 }
3616
3617 #ifndef multdiv_assemble
3618 void multdiv_assemble(int i,struct regstat *i_regs)
3619 {
3620   printf("Need multdiv_assemble for this architecture.\n");
3621   abort();
3622 }
3623 #endif
3624
3625 static void mov_assemble(int i,struct regstat *i_regs)
3626 {
3627   //if(opcode2[i]==0x10||opcode2[i]==0x12) { // MFHI/MFLO
3628   //if(opcode2[i]==0x11||opcode2[i]==0x13) { // MTHI/MTLO
3629   if(rt1[i]) {
3630     signed char sl,tl;
3631     tl=get_reg(i_regs->regmap,rt1[i]);
3632     //assert(tl>=0);
3633     if(tl>=0) {
3634       sl=get_reg(i_regs->regmap,rs1[i]);
3635       if(sl>=0) emit_mov(sl,tl);
3636       else emit_loadreg(rs1[i],tl);
3637     }
3638   }
3639 }
3640
3641 // call interpreter, exception handler, things that change pc/regs/cycles ...
3642 static void call_c_cpu_handler(int i, const struct regstat *i_regs, u_int pc, void *func)
3643 {
3644   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3645   assert(ccreg==HOST_CCREG);
3646   assert(!is_delayslot);
3647   (void)ccreg;
3648
3649   emit_movimm(pc,3); // Get PC
3650   emit_readword(&last_count,2);
3651   emit_writeword(3,&psxRegs.pc);
3652   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // XXX
3653   emit_add(2,HOST_CCREG,2);
3654   emit_writeword(2,&psxRegs.cycle);
3655   emit_far_call(func);
3656   emit_far_jump(jump_to_new_pc);
3657 }
3658
3659 static void syscall_assemble(int i,struct regstat *i_regs)
3660 {
3661   emit_movimm(0x20,0); // cause code
3662   emit_movimm(0,1);    // not in delay slot
3663   call_c_cpu_handler(i,i_regs,start+i*4,psxException);
3664 }
3665
3666 static void hlecall_assemble(int i,struct regstat *i_regs)
3667 {
3668   void *hlefunc = psxNULL;
3669   uint32_t hleCode = source[i] & 0x03ffffff;
3670   if (hleCode < ARRAY_SIZE(psxHLEt))
3671     hlefunc = psxHLEt[hleCode];
3672
3673   call_c_cpu_handler(i,i_regs,start+i*4+4,hlefunc);
3674 }
3675
3676 static void intcall_assemble(int i,struct regstat *i_regs)
3677 {
3678   call_c_cpu_handler(i,i_regs,start+i*4,execI);
3679 }
3680
3681 static void speculate_mov(int rs,int rt)
3682 {
3683   if(rt!=0) {
3684     smrv_strong_next|=1<<rt;
3685     smrv[rt]=smrv[rs];
3686   }
3687 }
3688
3689 static void speculate_mov_weak(int rs,int rt)
3690 {
3691   if(rt!=0) {
3692     smrv_weak_next|=1<<rt;
3693     smrv[rt]=smrv[rs];
3694   }
3695 }
3696
3697 static void speculate_register_values(int i)
3698 {
3699   if(i==0) {
3700     memcpy(smrv,psxRegs.GPR.r,sizeof(smrv));
3701     // gp,sp are likely to stay the same throughout the block
3702     smrv_strong_next=(1<<28)|(1<<29)|(1<<30);
3703     smrv_weak_next=~smrv_strong_next;
3704     //printf(" llr %08x\n", smrv[4]);
3705   }
3706   smrv_strong=smrv_strong_next;
3707   smrv_weak=smrv_weak_next;
3708   switch(itype[i]) {
3709     case ALU:
3710       if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3711       else if((smrv_strong>>rs2[i])&1) speculate_mov(rs2[i],rt1[i]);
3712       else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3713       else if((smrv_weak>>rs2[i])&1) speculate_mov_weak(rs2[i],rt1[i]);
3714       else {
3715         smrv_strong_next&=~(1<<rt1[i]);
3716         smrv_weak_next&=~(1<<rt1[i]);
3717       }
3718       break;
3719     case SHIFTIMM:
3720       smrv_strong_next&=~(1<<rt1[i]);
3721       smrv_weak_next&=~(1<<rt1[i]);
3722       // fallthrough
3723     case IMM16:
3724       if(rt1[i]&&is_const(&regs[i],rt1[i])) {
3725         int value,hr=get_reg(regs[i].regmap,rt1[i]);
3726         if(hr>=0) {
3727           if(get_final_value(hr,i,&value))
3728                smrv[rt1[i]]=value;
3729           else smrv[rt1[i]]=constmap[i][hr];
3730           smrv_strong_next|=1<<rt1[i];
3731         }
3732       }
3733       else {
3734         if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3735         else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3736       }
3737       break;
3738     case LOAD:
3739       if(start<0x2000&&(rt1[i]==26||(smrv[rt1[i]]>>24)==0xa0)) {
3740         // special case for BIOS
3741         smrv[rt1[i]]=0xa0000000;
3742         smrv_strong_next|=1<<rt1[i];
3743         break;
3744       }
3745       // fallthrough
3746     case SHIFT:
3747     case LOADLR:
3748     case MOV:
3749       smrv_strong_next&=~(1<<rt1[i]);
3750       smrv_weak_next&=~(1<<rt1[i]);
3751       break;
3752     case COP0:
3753     case COP2:
3754       if(opcode2[i]==0||opcode2[i]==2) { // MFC/CFC
3755         smrv_strong_next&=~(1<<rt1[i]);
3756         smrv_weak_next&=~(1<<rt1[i]);
3757       }
3758       break;
3759     case C2LS:
3760       if (opcode[i]==0x32) { // LWC2
3761         smrv_strong_next&=~(1<<rt1[i]);
3762         smrv_weak_next&=~(1<<rt1[i]);
3763       }
3764       break;
3765   }
3766 #if 0
3767   int r=4;
3768   printf("x %08x %08x %d %d c %08x %08x\n",smrv[r],start+i*4,
3769     ((smrv_strong>>r)&1),(smrv_weak>>r)&1,regs[i].isconst,regs[i].wasconst);
3770 #endif
3771 }
3772
3773 static void ds_assemble(int i,struct regstat *i_regs)
3774 {
3775   speculate_register_values(i);
3776   is_delayslot=1;
3777   switch(itype[i]) {
3778     case ALU:
3779       alu_assemble(i,i_regs);break;
3780     case IMM16:
3781       imm16_assemble(i,i_regs);break;
3782     case SHIFT:
3783       shift_assemble(i,i_regs);break;
3784     case SHIFTIMM:
3785       shiftimm_assemble(i,i_regs);break;
3786     case LOAD:
3787       load_assemble(i,i_regs);break;
3788     case LOADLR:
3789       loadlr_assemble(i,i_regs);break;
3790     case STORE:
3791       store_assemble(i,i_regs);break;
3792     case STORELR:
3793       storelr_assemble(i,i_regs);break;
3794     case COP0:
3795       cop0_assemble(i,i_regs);break;
3796     case COP1:
3797       cop1_assemble(i,i_regs);break;
3798     case C1LS:
3799       c1ls_assemble(i,i_regs);break;
3800     case COP2:
3801       cop2_assemble(i,i_regs);break;
3802     case C2LS:
3803       c2ls_assemble(i,i_regs);break;
3804     case C2OP:
3805       c2op_assemble(i,i_regs);break;
3806     case MULTDIV:
3807       multdiv_assemble(i,i_regs);break;
3808     case MOV:
3809       mov_assemble(i,i_regs);break;
3810     case SYSCALL:
3811     case HLECALL:
3812     case INTCALL:
3813     case SPAN:
3814     case UJUMP:
3815     case RJUMP:
3816     case CJUMP:
3817     case SJUMP:
3818       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
3819   }
3820   is_delayslot=0;
3821 }
3822
3823 // Is the branch target a valid internal jump?
3824 static int internal_branch(int addr)
3825 {
3826   if(addr&1) return 0; // Indirect (register) jump
3827   if(addr>=start && addr<start+slen*4-4)
3828   {
3829     return 1;
3830   }
3831   return 0;
3832 }
3833
3834 static void wb_invalidate(signed char pre[],signed char entry[],uint64_t dirty,uint64_t u)
3835 {
3836   int hr;
3837   for(hr=0;hr<HOST_REGS;hr++) {
3838     if(hr!=EXCLUDE_REG) {
3839       if(pre[hr]!=entry[hr]) {
3840         if(pre[hr]>=0) {
3841           if((dirty>>hr)&1) {
3842             if(get_reg(entry,pre[hr])<0) {
3843               assert(pre[hr]<64);
3844               if(!((u>>pre[hr])&1))
3845                 emit_storereg(pre[hr],hr);
3846             }
3847           }
3848         }
3849       }
3850     }
3851   }
3852   // Move from one register to another (no writeback)
3853   for(hr=0;hr<HOST_REGS;hr++) {
3854     if(hr!=EXCLUDE_REG) {
3855       if(pre[hr]!=entry[hr]) {
3856         if(pre[hr]>=0&&(pre[hr]&63)<TEMPREG) {
3857           int nr;
3858           if((nr=get_reg(entry,pre[hr]))>=0) {
3859             emit_mov(hr,nr);
3860           }
3861         }
3862       }
3863     }
3864   }
3865 }
3866
3867 // Load the specified registers
3868 // This only loads the registers given as arguments because
3869 // we don't want to load things that will be overwritten
3870 static void load_regs(signed char entry[],signed char regmap[],int rs1,int rs2)
3871 {
3872   int hr;
3873   // Load 32-bit regs
3874   for(hr=0;hr<HOST_REGS;hr++) {
3875     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3876       if(entry[hr]!=regmap[hr]) {
3877         if(regmap[hr]==rs1||regmap[hr]==rs2)
3878         {
3879           if(regmap[hr]==0) {
3880             emit_zeroreg(hr);
3881           }
3882           else
3883           {
3884             emit_loadreg(regmap[hr],hr);
3885           }
3886         }
3887       }
3888     }
3889   }
3890 }
3891
3892 // Load registers prior to the start of a loop
3893 // so that they are not loaded within the loop
3894 static void loop_preload(signed char pre[],signed char entry[])
3895 {
3896   int hr;
3897   for(hr=0;hr<HOST_REGS;hr++) {
3898     if(hr!=EXCLUDE_REG) {
3899       if(pre[hr]!=entry[hr]) {
3900         if(entry[hr]>=0) {
3901           if(get_reg(pre,entry[hr])<0) {
3902             assem_debug("loop preload:\n");
3903             //printf("loop preload: %d\n",hr);
3904             if(entry[hr]==0) {
3905               emit_zeroreg(hr);
3906             }
3907             else if(entry[hr]<TEMPREG)
3908             {
3909               emit_loadreg(entry[hr],hr);
3910             }
3911             else if(entry[hr]-64<TEMPREG)
3912             {
3913               emit_loadreg(entry[hr],hr);
3914             }
3915           }
3916         }
3917       }
3918     }
3919   }
3920 }
3921
3922 // Generate address for load/store instruction
3923 // goes to AGEN for writes, FTEMP for LOADLR and cop1/2 loads
3924 void address_generation(int i,struct regstat *i_regs,signed char entry[])
3925 {
3926   if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS||itype[i]==C2LS) {
3927     int ra=-1;
3928     int agr=AGEN1+(i&1);
3929     if(itype[i]==LOAD) {
3930       ra=get_reg(i_regs->regmap,rt1[i]);
3931       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3932       assert(ra>=0);
3933     }
3934     if(itype[i]==LOADLR) {
3935       ra=get_reg(i_regs->regmap,FTEMP);
3936     }
3937     if(itype[i]==STORE||itype[i]==STORELR) {
3938       ra=get_reg(i_regs->regmap,agr);
3939       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3940     }
3941     if(itype[i]==C1LS||itype[i]==C2LS) {
3942       if ((opcode[i]&0x3b)==0x31||(opcode[i]&0x3b)==0x32) // LWC1/LDC1/LWC2/LDC2
3943         ra=get_reg(i_regs->regmap,FTEMP);
3944       else { // SWC1/SDC1/SWC2/SDC2
3945         ra=get_reg(i_regs->regmap,agr);
3946         if(ra<0) ra=get_reg(i_regs->regmap,-1);
3947       }
3948     }
3949     int rs=get_reg(i_regs->regmap,rs1[i]);
3950     if(ra>=0) {
3951       int offset=imm[i];
3952       int c=(i_regs->wasconst>>rs)&1;
3953       if(rs1[i]==0) {
3954         // Using r0 as a base address
3955         if(!entry||entry[ra]!=agr) {
3956           if (opcode[i]==0x22||opcode[i]==0x26) {
3957             emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3958           }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3959             emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3960           }else{
3961             emit_movimm(offset,ra);
3962           }
3963         } // else did it in the previous cycle
3964       }
3965       else if(rs<0) {
3966         if(!entry||entry[ra]!=rs1[i])
3967           emit_loadreg(rs1[i],ra);
3968         //if(!entry||entry[ra]!=rs1[i])
3969         //  printf("poor load scheduling!\n");
3970       }
3971       else if(c) {
3972         if(rs1[i]!=rt1[i]||itype[i]!=LOAD) {
3973           if(!entry||entry[ra]!=agr) {
3974             if (opcode[i]==0x22||opcode[i]==0x26) {
3975               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3976             }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3977               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3978             }else{
3979               emit_movimm(constmap[i][rs]+offset,ra);
3980               regs[i].loadedconst|=1<<ra;
3981             }
3982           } // else did it in the previous cycle
3983         } // else load_consts already did it
3984       }
3985       if(offset&&!c&&rs1[i]) {
3986         if(rs>=0) {
3987           emit_addimm(rs,offset,ra);
3988         }else{
3989           emit_addimm(ra,offset,ra);
3990         }
3991       }
3992     }
3993   }
3994   // Preload constants for next instruction
3995   if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS||itype[i+1]==C2LS) {
3996     int agr,ra;
3997     // Actual address
3998     agr=AGEN1+((i+1)&1);
3999     ra=get_reg(i_regs->regmap,agr);
4000     if(ra>=0) {
4001       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
4002       int offset=imm[i+1];
4003       int c=(regs[i+1].wasconst>>rs)&1;
4004       if(c&&(rs1[i+1]!=rt1[i+1]||itype[i+1]!=LOAD)) {
4005         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
4006           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
4007         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
4008           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
4009         }else{
4010           emit_movimm(constmap[i+1][rs]+offset,ra);
4011           regs[i+1].loadedconst|=1<<ra;
4012         }
4013       }
4014       else if(rs1[i+1]==0) {
4015         // Using r0 as a base address
4016         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
4017           emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
4018         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
4019           emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
4020         }else{
4021           emit_movimm(offset,ra);
4022         }
4023       }
4024     }
4025   }
4026 }
4027
4028 static int get_final_value(int hr, int i, int *value)
4029 {
4030   int reg=regs[i].regmap[hr];
4031   while(i<slen-1) {
4032     if(regs[i+1].regmap[hr]!=reg) break;
4033     if(!((regs[i+1].isconst>>hr)&1)) break;
4034     if(bt[i+1]) break;
4035     i++;
4036   }
4037   if(i<slen-1) {
4038     if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
4039       *value=constmap[i][hr];
4040       return 1;
4041     }
4042     if(!bt[i+1]) {
4043       if(itype[i+1]==UJUMP||itype[i+1]==RJUMP||itype[i+1]==CJUMP||itype[i+1]==SJUMP) {
4044         // Load in delay slot, out-of-order execution
4045         if(itype[i+2]==LOAD&&rs1[i+2]==reg&&rt1[i+2]==reg&&((regs[i+1].wasconst>>hr)&1))
4046         {
4047           // Precompute load address
4048           *value=constmap[i][hr]+imm[i+2];
4049           return 1;
4050         }
4051       }
4052       if(itype[i+1]==LOAD&&rs1[i+1]==reg&&rt1[i+1]==reg)
4053       {
4054         // Precompute load address
4055         *value=constmap[i][hr]+imm[i+1];
4056         //printf("c=%x imm=%lx\n",(long)constmap[i][hr],imm[i+1]);
4057         return 1;
4058       }
4059     }
4060   }
4061   *value=constmap[i][hr];
4062   //printf("c=%lx\n",(long)constmap[i][hr]);
4063   if(i==slen-1) return 1;
4064   assert(reg < 64);
4065   return !((unneeded_reg[i+1]>>reg)&1);
4066 }
4067
4068 // Load registers with known constants
4069 static void load_consts(signed char pre[],signed char regmap[],int i)
4070 {
4071   int hr,hr2;
4072   // propagate loaded constant flags
4073   if(i==0||bt[i])
4074     regs[i].loadedconst=0;
4075   else {
4076     for(hr=0;hr<HOST_REGS;hr++) {
4077       if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((regs[i-1].isconst>>hr)&1)&&pre[hr]==regmap[hr]
4078          &&regmap[hr]==regs[i-1].regmap[hr]&&((regs[i-1].loadedconst>>hr)&1))
4079       {
4080         regs[i].loadedconst|=1<<hr;
4081       }
4082     }
4083   }
4084   // Load 32-bit regs
4085   for(hr=0;hr<HOST_REGS;hr++) {
4086     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
4087       //if(entry[hr]!=regmap[hr]) {
4088       if(!((regs[i].loadedconst>>hr)&1)) {
4089         assert(regmap[hr]<64);
4090         if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
4091           int value,similar=0;
4092           if(get_final_value(hr,i,&value)) {
4093             // see if some other register has similar value
4094             for(hr2=0;hr2<HOST_REGS;hr2++) {
4095               if(hr2!=EXCLUDE_REG&&((regs[i].loadedconst>>hr2)&1)) {
4096                 if(is_similar_value(value,constmap[i][hr2])) {
4097                   similar=1;
4098                   break;
4099                 }
4100               }
4101             }
4102             if(similar) {
4103               int value2;
4104               if(get_final_value(hr2,i,&value2)) // is this needed?
4105                 emit_movimm_from(value2,hr2,value,hr);
4106               else
4107                 emit_movimm(value,hr);
4108             }
4109             else if(value==0) {
4110               emit_zeroreg(hr);
4111             }
4112             else {
4113               emit_movimm(value,hr);
4114             }
4115           }
4116           regs[i].loadedconst|=1<<hr;
4117         }
4118       }
4119     }
4120   }
4121 }
4122
4123 void load_all_consts(signed char regmap[], u_int dirty, int i)
4124 {
4125   int hr;
4126   // Load 32-bit regs
4127   for(hr=0;hr<HOST_REGS;hr++) {
4128     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
4129       assert(regmap[hr] < 64);
4130       if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
4131         int value=constmap[i][hr];
4132         if(value==0) {
4133           emit_zeroreg(hr);
4134         }
4135         else {
4136           emit_movimm(value,hr);
4137         }
4138       }
4139     }
4140   }
4141 }
4142
4143 // Write out all dirty registers (except cycle count)
4144 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty)
4145 {
4146   int hr;
4147   for(hr=0;hr<HOST_REGS;hr++) {
4148     if(hr!=EXCLUDE_REG) {
4149       if(i_regmap[hr]>0) {
4150         if(i_regmap[hr]!=CCREG) {
4151           if((i_dirty>>hr)&1) {
4152             assert(i_regmap[hr]<64);
4153             emit_storereg(i_regmap[hr],hr);
4154           }
4155         }
4156       }
4157     }
4158   }
4159 }
4160
4161 // Write out dirty registers that we need to reload (pair with load_needed_regs)
4162 // This writes the registers not written by store_regs_bt
4163 void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr)
4164 {
4165   int hr;
4166   int t=(addr-start)>>2;
4167   for(hr=0;hr<HOST_REGS;hr++) {
4168     if(hr!=EXCLUDE_REG) {
4169       if(i_regmap[hr]>0) {
4170         if(i_regmap[hr]!=CCREG) {
4171           if(i_regmap[hr]==regs[t].regmap_entry[hr] && ((regs[t].dirty>>hr)&1)) {
4172             if((i_dirty>>hr)&1) {
4173               assert(i_regmap[hr]<64);
4174               emit_storereg(i_regmap[hr],hr);
4175             }
4176           }
4177         }
4178       }
4179     }
4180   }
4181 }
4182
4183 // Load all registers (except cycle count)
4184 void load_all_regs(signed char i_regmap[])
4185 {
4186   int hr;
4187   for(hr=0;hr<HOST_REGS;hr++) {
4188     if(hr!=EXCLUDE_REG) {
4189       if(i_regmap[hr]==0) {
4190         emit_zeroreg(hr);
4191       }
4192       else
4193       if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4194       {
4195         emit_loadreg(i_regmap[hr],hr);
4196       }
4197     }
4198   }
4199 }
4200
4201 // Load all current registers also needed by next instruction
4202 void load_needed_regs(signed char i_regmap[],signed char next_regmap[])
4203 {
4204   int hr;
4205   for(hr=0;hr<HOST_REGS;hr++) {
4206     if(hr!=EXCLUDE_REG) {
4207       if(get_reg(next_regmap,i_regmap[hr])>=0) {
4208         if(i_regmap[hr]==0) {
4209           emit_zeroreg(hr);
4210         }
4211         else
4212         if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4213         {
4214           emit_loadreg(i_regmap[hr],hr);
4215         }
4216       }
4217     }
4218   }
4219 }
4220
4221 // Load all regs, storing cycle count if necessary
4222 void load_regs_entry(int t)
4223 {
4224   int hr;
4225   if(is_ds[t]) emit_addimm(HOST_CCREG,CLOCK_ADJUST(1),HOST_CCREG);
4226   else if(ccadj[t]) emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[t]),HOST_CCREG);
4227   if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4228     emit_storereg(CCREG,HOST_CCREG);
4229   }
4230   // Load 32-bit regs
4231   for(hr=0;hr<HOST_REGS;hr++) {
4232     if(regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4233       if(regs[t].regmap_entry[hr]==0) {
4234         emit_zeroreg(hr);
4235       }
4236       else if(regs[t].regmap_entry[hr]!=CCREG)
4237       {
4238         emit_loadreg(regs[t].regmap_entry[hr],hr);
4239       }
4240     }
4241   }
4242 }
4243
4244 // Store dirty registers prior to branch
4245 void store_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4246 {
4247   if(internal_branch(addr))
4248   {
4249     int t=(addr-start)>>2;
4250     int hr;
4251     for(hr=0;hr<HOST_REGS;hr++) {
4252       if(hr!=EXCLUDE_REG) {
4253         if(i_regmap[hr]>0 && i_regmap[hr]!=CCREG) {
4254           if(i_regmap[hr]!=regs[t].regmap_entry[hr] || !((regs[t].dirty>>hr)&1)) {
4255             if((i_dirty>>hr)&1) {
4256               assert(i_regmap[hr]<64);
4257               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4258                 emit_storereg(i_regmap[hr],hr);
4259             }
4260           }
4261         }
4262       }
4263     }
4264   }
4265   else
4266   {
4267     // Branch out of this block, write out all dirty regs
4268     wb_dirtys(i_regmap,i_dirty);
4269   }
4270 }
4271
4272 // Load all needed registers for branch target
4273 static void load_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4274 {
4275   //if(addr>=start && addr<(start+slen*4))
4276   if(internal_branch(addr))
4277   {
4278     int t=(addr-start)>>2;
4279     int hr;
4280     // Store the cycle count before loading something else
4281     if(i_regmap[HOST_CCREG]!=CCREG) {
4282       assert(i_regmap[HOST_CCREG]==-1);
4283     }
4284     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4285       emit_storereg(CCREG,HOST_CCREG);
4286     }
4287     // Load 32-bit regs
4288     for(hr=0;hr<HOST_REGS;hr++) {
4289       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4290         if(i_regmap[hr]!=regs[t].regmap_entry[hr]) {
4291           if(regs[t].regmap_entry[hr]==0) {
4292             emit_zeroreg(hr);
4293           }
4294           else if(regs[t].regmap_entry[hr]!=CCREG)
4295           {
4296             emit_loadreg(regs[t].regmap_entry[hr],hr);
4297           }
4298         }
4299       }
4300     }
4301   }
4302 }
4303
4304 static int match_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4305 {
4306   if(addr>=start && addr<start+slen*4-4)
4307   {
4308     int t=(addr-start)>>2;
4309     int hr;
4310     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) return 0;
4311     for(hr=0;hr<HOST_REGS;hr++)
4312     {
4313       if(hr!=EXCLUDE_REG)
4314       {
4315         if(i_regmap[hr]!=regs[t].regmap_entry[hr])
4316         {
4317           if(regs[t].regmap_entry[hr]>=0&&(regs[t].regmap_entry[hr]|64)<TEMPREG+64)
4318           {
4319             return 0;
4320           }
4321           else
4322           if((i_dirty>>hr)&1)
4323           {
4324             if(i_regmap[hr]<TEMPREG)
4325             {
4326               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4327                 return 0;
4328             }
4329             else if(i_regmap[hr]>=64&&i_regmap[hr]<TEMPREG+64)
4330             {
4331               assert(0);
4332             }
4333           }
4334         }
4335         else // Same register but is it 32-bit or dirty?
4336         if(i_regmap[hr]>=0)
4337         {
4338           if(!((regs[t].dirty>>hr)&1))
4339           {
4340             if((i_dirty>>hr)&1)
4341             {
4342               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4343               {
4344                 //printf("%x: dirty no match\n",addr);
4345                 return 0;
4346               }
4347             }
4348           }
4349         }
4350       }
4351     }
4352     // Delay slots are not valid branch targets
4353     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP)) return 0;
4354     // Delay slots require additional processing, so do not match
4355     if(is_ds[t]) return 0;
4356   }
4357   else
4358   {
4359     int hr;
4360     for(hr=0;hr<HOST_REGS;hr++)
4361     {
4362       if(hr!=EXCLUDE_REG)
4363       {
4364         if(i_regmap[hr]>=0)
4365         {
4366           if(hr!=HOST_CCREG||i_regmap[hr]!=CCREG)
4367           {
4368             if((i_dirty>>hr)&1)
4369             {
4370               return 0;
4371             }
4372           }
4373         }
4374       }
4375     }
4376   }
4377   return 1;
4378 }
4379
4380 #ifdef DRC_DBG
4381 static void drc_dbg_emit_do_cmp(int i)
4382 {
4383   extern void do_insn_cmp();
4384   //extern int cycle;
4385   u_int hr, reglist = get_host_reglist(regs[i].regmap);
4386
4387   assem_debug("//do_insn_cmp %08x\n", start+i*4);
4388   save_regs(reglist);
4389   // write out changed consts to match the interpreter
4390   if (i > 0 && !bt[i]) {
4391     for (hr = 0; hr < HOST_REGS; hr++) {
4392       int reg = regs[i-1].regmap[hr];
4393       if (hr == EXCLUDE_REG || reg < 0)
4394         continue;
4395       if (!((regs[i-1].isconst >> hr) & 1))
4396         continue;
4397       if (i > 1 && reg == regs[i-2].regmap[hr] && constmap[i-1][hr] == constmap[i-2][hr])
4398         continue;
4399       emit_movimm(constmap[i-1][hr],0);
4400       emit_storereg(reg, 0);
4401     }
4402   }
4403   emit_movimm(start+i*4,0);
4404   emit_writeword(0,&pcaddr);
4405   emit_far_call(do_insn_cmp);
4406   //emit_readword(&cycle,0);
4407   //emit_addimm(0,2,0);
4408   //emit_writeword(0,&cycle);
4409   (void)get_reg2;
4410   restore_regs(reglist);
4411   assem_debug("\\\\do_insn_cmp\n");
4412 }
4413 #else
4414 #define drc_dbg_emit_do_cmp(x)
4415 #endif
4416
4417 // Used when a branch jumps into the delay slot of another branch
4418 static void ds_assemble_entry(int i)
4419 {
4420   int t=(ba[i]-start)>>2;
4421   if (!instr_addr[t])
4422     instr_addr[t] = out;
4423   assem_debug("Assemble delay slot at %x\n",ba[i]);
4424   assem_debug("<->\n");
4425   drc_dbg_emit_do_cmp(t);
4426   if(regs[t].regmap_entry[HOST_CCREG]==CCREG&&regs[t].regmap[HOST_CCREG]!=CCREG)
4427     wb_register(CCREG,regs[t].regmap_entry,regs[t].wasdirty);
4428   load_regs(regs[t].regmap_entry,regs[t].regmap,rs1[t],rs2[t]);
4429   address_generation(t,&regs[t],regs[t].regmap_entry);
4430   if(itype[t]==STORE||itype[t]==STORELR||(opcode[t]&0x3b)==0x39||(opcode[t]&0x3b)==0x3a)
4431     load_regs(regs[t].regmap_entry,regs[t].regmap,INVCP,INVCP);
4432   is_delayslot=0;
4433   switch(itype[t]) {
4434     case ALU:
4435       alu_assemble(t,&regs[t]);break;
4436     case IMM16:
4437       imm16_assemble(t,&regs[t]);break;
4438     case SHIFT:
4439       shift_assemble(t,&regs[t]);break;
4440     case SHIFTIMM:
4441       shiftimm_assemble(t,&regs[t]);break;
4442     case LOAD:
4443       load_assemble(t,&regs[t]);break;
4444     case LOADLR:
4445       loadlr_assemble(t,&regs[t]);break;
4446     case STORE:
4447       store_assemble(t,&regs[t]);break;
4448     case STORELR:
4449       storelr_assemble(t,&regs[t]);break;
4450     case COP0:
4451       cop0_assemble(t,&regs[t]);break;
4452     case COP1:
4453       cop1_assemble(t,&regs[t]);break;
4454     case C1LS:
4455       c1ls_assemble(t,&regs[t]);break;
4456     case COP2:
4457       cop2_assemble(t,&regs[t]);break;
4458     case C2LS:
4459       c2ls_assemble(t,&regs[t]);break;
4460     case C2OP:
4461       c2op_assemble(t,&regs[t]);break;
4462     case MULTDIV:
4463       multdiv_assemble(t,&regs[t]);break;
4464     case MOV:
4465       mov_assemble(t,&regs[t]);break;
4466     case SYSCALL:
4467     case HLECALL:
4468     case INTCALL:
4469     case SPAN:
4470     case UJUMP:
4471     case RJUMP:
4472     case CJUMP:
4473     case SJUMP:
4474       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
4475   }
4476   store_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4477   load_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4478   if(internal_branch(ba[i]+4))
4479     assem_debug("branch: internal\n");
4480   else
4481     assem_debug("branch: external\n");
4482   assert(internal_branch(ba[i]+4));
4483   add_to_linker(out,ba[i]+4,internal_branch(ba[i]+4));
4484   emit_jmp(0);
4485 }
4486
4487 static void emit_extjump(void *addr, u_int target)
4488 {
4489   emit_extjump2(addr, target, dyna_linker);
4490 }
4491
4492 static void emit_extjump_ds(void *addr, u_int target)
4493 {
4494   emit_extjump2(addr, target, dyna_linker_ds);
4495 }
4496
4497 // Load 2 immediates optimizing for small code size
4498 static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2)
4499 {
4500   emit_movimm(imm1,rt1);
4501   emit_movimm_from(imm1,rt1,imm2,rt2);
4502 }
4503
4504 void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert)
4505 {
4506   int count;
4507   void *jaddr;
4508   void *idle=NULL;
4509   int t=0;
4510   if(itype[i]==RJUMP)
4511   {
4512     *adj=0;
4513   }
4514   //if(ba[i]>=start && ba[i]<(start+slen*4))
4515   if(internal_branch(ba[i]))
4516   {
4517     t=(ba[i]-start)>>2;
4518     if(is_ds[t]) *adj=-1; // Branch into delay slot adds an extra cycle
4519     else *adj=ccadj[t];
4520   }
4521   else
4522   {
4523     *adj=0;
4524   }
4525   count=ccadj[i];
4526   if(taken==TAKEN && i==(ba[i]-start)>>2 && source[i+1]==0) {
4527     // Idle loop
4528     if(count&1) emit_addimm_and_set_flags(2*(count+2),HOST_CCREG);
4529     idle=out;
4530     //emit_subfrommem(&idlecount,HOST_CCREG); // Count idle cycles
4531     emit_andimm(HOST_CCREG,3,HOST_CCREG);
4532     jaddr=out;
4533     emit_jmp(0);
4534   }
4535   else if(*adj==0||invert) {
4536     int cycles=CLOCK_ADJUST(count+2);
4537     // faster loop HACK
4538 #if 0
4539     if (t&&*adj) {
4540       int rel=t-i;
4541       if(-NO_CYCLE_PENALTY_THR<rel&&rel<0)
4542         cycles=CLOCK_ADJUST(*adj)+count+2-*adj;
4543     }
4544 #endif
4545     emit_addimm_and_set_flags(cycles,HOST_CCREG);
4546     jaddr=out;
4547     emit_jns(0);
4548   }
4549   else
4550   {
4551     emit_cmpimm(HOST_CCREG,-CLOCK_ADJUST(count+2));
4552     jaddr=out;
4553     emit_jns(0);
4554   }
4555   add_stub(CC_STUB,jaddr,idle?idle:out,(*adj==0||invert||idle)?0:(count+2),i,addr,taken,0);
4556 }
4557
4558 static void do_ccstub(int n)
4559 {
4560   literal_pool(256);
4561   assem_debug("do_ccstub %x\n",start+(u_int)stubs[n].b*4);
4562   set_jump_target(stubs[n].addr, out);
4563   int i=stubs[n].b;
4564   if(stubs[n].d==NULLDS) {
4565     // Delay slot instruction is nullified ("likely" branch)
4566     wb_dirtys(regs[i].regmap,regs[i].dirty);
4567   }
4568   else if(stubs[n].d!=TAKEN) {
4569     wb_dirtys(branch_regs[i].regmap,branch_regs[i].dirty);
4570   }
4571   else {
4572     if(internal_branch(ba[i]))
4573       wb_needed_dirtys(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4574   }
4575   if(stubs[n].c!=-1)
4576   {
4577     // Save PC as return address
4578     emit_movimm(stubs[n].c,EAX);
4579     emit_writeword(EAX,&pcaddr);
4580   }
4581   else
4582   {
4583     // Return address depends on which way the branch goes
4584     if(itype[i]==CJUMP||itype[i]==SJUMP)
4585     {
4586       int s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4587       int s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4588       if(rs1[i]==0)
4589       {
4590         s1l=s2l;
4591         s2l=-1;
4592       }
4593       else if(rs2[i]==0)
4594       {
4595         s2l=-1;
4596       }
4597       assert(s1l>=0);
4598       #ifdef DESTRUCTIVE_WRITEBACK
4599       if(rs1[i]) {
4600         if((branch_regs[i].dirty>>s1l)&&1)
4601           emit_loadreg(rs1[i],s1l);
4602       }
4603       else {
4604         if((branch_regs[i].dirty>>s1l)&1)
4605           emit_loadreg(rs2[i],s1l);
4606       }
4607       if(s2l>=0)
4608         if((branch_regs[i].dirty>>s2l)&1)
4609           emit_loadreg(rs2[i],s2l);
4610       #endif
4611       int hr=0;
4612       int addr=-1,alt=-1,ntaddr=-1;
4613       while(hr<HOST_REGS)
4614       {
4615         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4616            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4617            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4618         {
4619           addr=hr++;break;
4620         }
4621         hr++;
4622       }
4623       while(hr<HOST_REGS)
4624       {
4625         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4626            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4627            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4628         {
4629           alt=hr++;break;
4630         }
4631         hr++;
4632       }
4633       if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
4634       {
4635         while(hr<HOST_REGS)
4636         {
4637           if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4638              (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4639              (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4640           {
4641             ntaddr=hr;break;
4642           }
4643           hr++;
4644         }
4645         assert(hr<HOST_REGS);
4646       }
4647       if((opcode[i]&0x2f)==4) // BEQ
4648       {
4649         #ifdef HAVE_CMOV_IMM
4650         if(s2l>=0) emit_cmp(s1l,s2l);
4651         else emit_test(s1l,s1l);
4652         emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
4653         #else
4654         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4655         if(s2l>=0) emit_cmp(s1l,s2l);
4656         else emit_test(s1l,s1l);
4657         emit_cmovne_reg(alt,addr);
4658         #endif
4659       }
4660       if((opcode[i]&0x2f)==5) // BNE
4661       {
4662         #ifdef HAVE_CMOV_IMM
4663         if(s2l>=0) emit_cmp(s1l,s2l);
4664         else emit_test(s1l,s1l);
4665         emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
4666         #else
4667         emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
4668         if(s2l>=0) emit_cmp(s1l,s2l);
4669         else emit_test(s1l,s1l);
4670         emit_cmovne_reg(alt,addr);
4671         #endif
4672       }
4673       if((opcode[i]&0x2f)==6) // BLEZ
4674       {
4675         //emit_movimm(ba[i],alt);
4676         //emit_movimm(start+i*4+8,addr);
4677         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4678         emit_cmpimm(s1l,1);
4679         emit_cmovl_reg(alt,addr);
4680       }
4681       if((opcode[i]&0x2f)==7) // BGTZ
4682       {
4683         //emit_movimm(ba[i],addr);
4684         //emit_movimm(start+i*4+8,ntaddr);
4685         emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
4686         emit_cmpimm(s1l,1);
4687         emit_cmovl_reg(ntaddr,addr);
4688       }
4689       if((opcode[i]==1)&&(opcode2[i]&0x2D)==0) // BLTZ
4690       {
4691         //emit_movimm(ba[i],alt);
4692         //emit_movimm(start+i*4+8,addr);
4693         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4694         emit_test(s1l,s1l);
4695         emit_cmovs_reg(alt,addr);
4696       }
4697       if((opcode[i]==1)&&(opcode2[i]&0x2D)==1) // BGEZ
4698       {
4699         //emit_movimm(ba[i],addr);
4700         //emit_movimm(start+i*4+8,alt);
4701         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4702         emit_test(s1l,s1l);
4703         emit_cmovs_reg(alt,addr);
4704       }
4705       if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
4706         if(source[i]&0x10000) // BC1T
4707         {
4708           //emit_movimm(ba[i],alt);
4709           //emit_movimm(start+i*4+8,addr);
4710           emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4711           emit_testimm(s1l,0x800000);
4712           emit_cmovne_reg(alt,addr);
4713         }
4714         else // BC1F
4715         {
4716           //emit_movimm(ba[i],addr);
4717           //emit_movimm(start+i*4+8,alt);
4718           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4719           emit_testimm(s1l,0x800000);
4720           emit_cmovne_reg(alt,addr);
4721         }
4722       }
4723       emit_writeword(addr,&pcaddr);
4724     }
4725     else
4726     if(itype[i]==RJUMP)
4727     {
4728       int r=get_reg(branch_regs[i].regmap,rs1[i]);
4729       if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4730         r=get_reg(branch_regs[i].regmap,RTEMP);
4731       }
4732       emit_writeword(r,&pcaddr);
4733     }
4734     else {SysPrintf("Unknown branch type in do_ccstub\n");abort();}
4735   }
4736   // Update cycle count
4737   assert(branch_regs[i].regmap[HOST_CCREG]==CCREG||branch_regs[i].regmap[HOST_CCREG]==-1);
4738   if(stubs[n].a) emit_addimm(HOST_CCREG,CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4739   emit_far_call(cc_interrupt);
4740   if(stubs[n].a) emit_addimm(HOST_CCREG,-CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4741   if(stubs[n].d==TAKEN) {
4742     if(internal_branch(ba[i]))
4743       load_needed_regs(branch_regs[i].regmap,regs[(ba[i]-start)>>2].regmap_entry);
4744     else if(itype[i]==RJUMP) {
4745       if(get_reg(branch_regs[i].regmap,RTEMP)>=0)
4746         emit_readword(&pcaddr,get_reg(branch_regs[i].regmap,RTEMP));
4747       else
4748         emit_loadreg(rs1[i],get_reg(branch_regs[i].regmap,rs1[i]));
4749     }
4750   }else if(stubs[n].d==NOTTAKEN) {
4751     if(i<slen-2) load_needed_regs(branch_regs[i].regmap,regmap_pre[i+2]);
4752     else load_all_regs(branch_regs[i].regmap);
4753   }else if(stubs[n].d==NULLDS) {
4754     // Delay slot instruction is nullified ("likely" branch)
4755     if(i<slen-2) load_needed_regs(regs[i].regmap,regmap_pre[i+2]);
4756     else load_all_regs(regs[i].regmap);
4757   }else{
4758     load_all_regs(branch_regs[i].regmap);
4759   }
4760   if (stubs[n].retaddr)
4761     emit_jmp(stubs[n].retaddr);
4762   else
4763     do_jump_vaddr(stubs[n].e);
4764 }
4765
4766 static void add_to_linker(void *addr, u_int target, int ext)
4767 {
4768   assert(linkcount < ARRAY_SIZE(link_addr));
4769   link_addr[linkcount].addr = addr;
4770   link_addr[linkcount].target = target;
4771   link_addr[linkcount].ext = ext;
4772   linkcount++;
4773 }
4774
4775 static void ujump_assemble_write_ra(int i)
4776 {
4777   int rt;
4778   unsigned int return_address;
4779   rt=get_reg(branch_regs[i].regmap,31);
4780   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4781   //assert(rt>=0);
4782   return_address=start+i*4+8;
4783   if(rt>=0) {
4784     #ifdef USE_MINI_HT
4785     if(internal_branch(return_address)&&rt1[i+1]!=31) {
4786       int temp=-1; // note: must be ds-safe
4787       #ifdef HOST_TEMPREG
4788       temp=HOST_TEMPREG;
4789       #endif
4790       if(temp>=0) do_miniht_insert(return_address,rt,temp);
4791       else emit_movimm(return_address,rt);
4792     }
4793     else
4794     #endif
4795     {
4796       #ifdef REG_PREFETCH
4797       if(temp>=0)
4798       {
4799         if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4800       }
4801       #endif
4802       emit_movimm(return_address,rt); // PC into link register
4803       #ifdef IMM_PREFETCH
4804       emit_prefetch(hash_table_get(return_address));
4805       #endif
4806     }
4807   }
4808 }
4809
4810 static void ujump_assemble(int i,struct regstat *i_regs)
4811 {
4812   int ra_done=0;
4813   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4814   address_generation(i+1,i_regs,regs[i].regmap_entry);
4815   #ifdef REG_PREFETCH
4816   int temp=get_reg(branch_regs[i].regmap,PTEMP);
4817   if(rt1[i]==31&&temp>=0)
4818   {
4819     signed char *i_regmap=i_regs->regmap;
4820     int return_address=start+i*4+8;
4821     if(get_reg(branch_regs[i].regmap,31)>0)
4822     if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4823   }
4824   #endif
4825   if(rt1[i]==31&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4826     ujump_assemble_write_ra(i); // writeback ra for DS
4827     ra_done=1;
4828   }
4829   ds_assemble(i+1,i_regs);
4830   uint64_t bc_unneeded=branch_regs[i].u;
4831   bc_unneeded|=1|(1LL<<rt1[i]);
4832   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4833   load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4834   if(!ra_done&&rt1[i]==31)
4835     ujump_assemble_write_ra(i);
4836   int cc,adj;
4837   cc=get_reg(branch_regs[i].regmap,CCREG);
4838   assert(cc==HOST_CCREG);
4839   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4840   #ifdef REG_PREFETCH
4841   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4842   #endif
4843   do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4844   if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4845   load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4846   if(internal_branch(ba[i]))
4847     assem_debug("branch: internal\n");
4848   else
4849     assem_debug("branch: external\n");
4850   if(internal_branch(ba[i])&&is_ds[(ba[i]-start)>>2]) {
4851     ds_assemble_entry(i);
4852   }
4853   else {
4854     add_to_linker(out,ba[i],internal_branch(ba[i]));
4855     emit_jmp(0);
4856   }
4857 }
4858
4859 static void rjump_assemble_write_ra(int i)
4860 {
4861   int rt,return_address;
4862   assert(rt1[i+1]!=rt1[i]);
4863   assert(rt2[i+1]!=rt1[i]);
4864   rt=get_reg(branch_regs[i].regmap,rt1[i]);
4865   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4866   assert(rt>=0);
4867   return_address=start+i*4+8;
4868   #ifdef REG_PREFETCH
4869   if(temp>=0)
4870   {
4871     if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4872   }
4873   #endif
4874   emit_movimm(return_address,rt); // PC into link register
4875   #ifdef IMM_PREFETCH
4876   emit_prefetch(hash_table_get(return_address));
4877   #endif
4878 }
4879
4880 static void rjump_assemble(int i,struct regstat *i_regs)
4881 {
4882   int temp;
4883   int rs,cc;
4884   int ra_done=0;
4885   rs=get_reg(branch_regs[i].regmap,rs1[i]);
4886   assert(rs>=0);
4887   if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4888     // Delay slot abuse, make a copy of the branch address register
4889     temp=get_reg(branch_regs[i].regmap,RTEMP);
4890     assert(temp>=0);
4891     assert(regs[i].regmap[temp]==RTEMP);
4892     emit_mov(rs,temp);
4893     rs=temp;
4894   }
4895   address_generation(i+1,i_regs,regs[i].regmap_entry);
4896   #ifdef REG_PREFETCH
4897   if(rt1[i]==31)
4898   {
4899     if((temp=get_reg(branch_regs[i].regmap,PTEMP))>=0) {
4900       signed char *i_regmap=i_regs->regmap;
4901       int return_address=start+i*4+8;
4902       if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4903     }
4904   }
4905   #endif
4906   #ifdef USE_MINI_HT
4907   if(rs1[i]==31) {
4908     int rh=get_reg(regs[i].regmap,RHASH);
4909     if(rh>=0) do_preload_rhash(rh);
4910   }
4911   #endif
4912   if(rt1[i]!=0&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4913     rjump_assemble_write_ra(i);
4914     ra_done=1;
4915   }
4916   ds_assemble(i+1,i_regs);
4917   uint64_t bc_unneeded=branch_regs[i].u;
4918   bc_unneeded|=1|(1LL<<rt1[i]);
4919   bc_unneeded&=~(1LL<<rs1[i]);
4920   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4921   load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],CCREG);
4922   if(!ra_done&&rt1[i]!=0)
4923     rjump_assemble_write_ra(i);
4924   cc=get_reg(branch_regs[i].regmap,CCREG);
4925   assert(cc==HOST_CCREG);
4926   (void)cc;
4927   #ifdef USE_MINI_HT
4928   int rh=get_reg(branch_regs[i].regmap,RHASH);
4929   int ht=get_reg(branch_regs[i].regmap,RHTBL);
4930   if(rs1[i]==31) {
4931     if(regs[i].regmap[rh]!=RHASH) do_preload_rhash(rh);
4932     do_preload_rhtbl(ht);
4933     do_rhash(rs,rh);
4934   }
4935   #endif
4936   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4937   #ifdef DESTRUCTIVE_WRITEBACK
4938   if((branch_regs[i].dirty>>rs)&1) {
4939     if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
4940       emit_loadreg(rs1[i],rs);
4941     }
4942   }
4943   #endif
4944   #ifdef REG_PREFETCH
4945   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4946   #endif
4947   #ifdef USE_MINI_HT
4948   if(rs1[i]==31) {
4949     do_miniht_load(ht,rh);
4950   }
4951   #endif
4952   //do_cc(i,branch_regs[i].regmap,&adj,-1,TAKEN);
4953   //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen
4954   //assert(adj==0);
4955   emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4956   add_stub(CC_STUB,out,NULL,0,i,-1,TAKEN,rs);
4957   if(itype[i+1]==COP0&&(source[i+1]&0x3f)==0x10)
4958     // special case for RFE
4959     emit_jmp(0);
4960   else
4961     emit_jns(0);
4962   //load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4963   #ifdef USE_MINI_HT
4964   if(rs1[i]==31) {
4965     do_miniht_jump(rs,rh,ht);
4966   }
4967   else
4968   #endif
4969   {
4970     do_jump_vaddr(rs);
4971   }
4972   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4973   if(rt1[i]!=31&&i<slen-2&&(((u_int)out)&7)) emit_mov(13,13);
4974   #endif
4975 }
4976
4977 static void cjump_assemble(int i,struct regstat *i_regs)
4978 {
4979   signed char *i_regmap=i_regs->regmap;
4980   int cc;
4981   int match;
4982   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4983   assem_debug("match=%d\n",match);
4984   int s1l,s2l;
4985   int unconditional=0,nop=0;
4986   int invert=0;
4987   int internal=internal_branch(ba[i]);
4988   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4989   if(!match) invert=1;
4990   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4991   if(i>(ba[i]-start)>>2) invert=1;
4992   #endif
4993   #ifdef __aarch64__
4994   invert=1; // because of near cond. branches
4995   #endif
4996
4997   if(ooo[i]) {
4998     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4999     s2l=get_reg(branch_regs[i].regmap,rs2[i]);
5000   }
5001   else {
5002     s1l=get_reg(i_regmap,rs1[i]);
5003     s2l=get_reg(i_regmap,rs2[i]);
5004   }
5005   if(rs1[i]==0&&rs2[i]==0)
5006   {
5007     if(opcode[i]&1) nop=1;
5008     else unconditional=1;
5009     //assert(opcode[i]!=5);
5010     //assert(opcode[i]!=7);
5011     //assert(opcode[i]!=0x15);
5012     //assert(opcode[i]!=0x17);
5013   }
5014   else if(rs1[i]==0)
5015   {
5016     s1l=s2l;
5017     s2l=-1;
5018   }
5019   else if(rs2[i]==0)
5020   {
5021     s2l=-1;
5022   }
5023
5024   if(ooo[i]) {
5025     // Out of order execution (delay slot first)
5026     //printf("OOOE\n");
5027     address_generation(i+1,i_regs,regs[i].regmap_entry);
5028     ds_assemble(i+1,i_regs);
5029     int adj;
5030     uint64_t bc_unneeded=branch_regs[i].u;
5031     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5032     bc_unneeded|=1;
5033     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
5034     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs2[i]);
5035     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5036     cc=get_reg(branch_regs[i].regmap,CCREG);
5037     assert(cc==HOST_CCREG);
5038     if(unconditional)
5039       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5040     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
5041     //assem_debug("cycle count (adj)\n");
5042     if(unconditional) {
5043       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5044       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
5045         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5046         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5047         if(internal)
5048           assem_debug("branch: internal\n");
5049         else
5050           assem_debug("branch: external\n");
5051         if(internal&&is_ds[(ba[i]-start)>>2]) {
5052           ds_assemble_entry(i);
5053         }
5054         else {
5055           add_to_linker(out,ba[i],internal);
5056           emit_jmp(0);
5057         }
5058         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5059         if(((u_int)out)&7) emit_addnop(0);
5060         #endif
5061       }
5062     }
5063     else if(nop) {
5064       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5065       void *jaddr=out;
5066       emit_jns(0);
5067       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5068     }
5069     else {
5070       void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
5071       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5072       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5073
5074       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5075       assert(s1l>=0);
5076       if(opcode[i]==4) // BEQ
5077       {
5078         if(s2l>=0) emit_cmp(s1l,s2l);
5079         else emit_test(s1l,s1l);
5080         if(invert){
5081           nottaken=out;
5082           emit_jne(DJT_1);
5083         }else{
5084           add_to_linker(out,ba[i],internal);
5085           emit_jeq(0);
5086         }
5087       }
5088       if(opcode[i]==5) // BNE
5089       {
5090         if(s2l>=0) emit_cmp(s1l,s2l);
5091         else emit_test(s1l,s1l);
5092         if(invert){
5093           nottaken=out;
5094           emit_jeq(DJT_1);
5095         }else{
5096           add_to_linker(out,ba[i],internal);
5097           emit_jne(0);
5098         }
5099       }
5100       if(opcode[i]==6) // BLEZ
5101       {
5102         emit_cmpimm(s1l,1);
5103         if(invert){
5104           nottaken=out;
5105           emit_jge(DJT_1);
5106         }else{
5107           add_to_linker(out,ba[i],internal);
5108           emit_jl(0);
5109         }
5110       }
5111       if(opcode[i]==7) // BGTZ
5112       {
5113         emit_cmpimm(s1l,1);
5114         if(invert){
5115           nottaken=out;
5116           emit_jl(DJT_1);
5117         }else{
5118           add_to_linker(out,ba[i],internal);
5119           emit_jge(0);
5120         }
5121       }
5122       if(invert) {
5123         if(taken) set_jump_target(taken, out);
5124         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5125         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5126           if(adj) {
5127             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5128             add_to_linker(out,ba[i],internal);
5129           }else{
5130             emit_addnop(13);
5131             add_to_linker(out,ba[i],internal*2);
5132           }
5133           emit_jmp(0);
5134         }else
5135         #endif
5136         {
5137           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5138           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5139           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5140           if(internal)
5141             assem_debug("branch: internal\n");
5142           else
5143             assem_debug("branch: external\n");
5144           if(internal&&is_ds[(ba[i]-start)>>2]) {
5145             ds_assemble_entry(i);
5146           }
5147           else {
5148             add_to_linker(out,ba[i],internal);
5149             emit_jmp(0);
5150           }
5151         }
5152         set_jump_target(nottaken, out);
5153       }
5154
5155       if(nottaken1) set_jump_target(nottaken1, out);
5156       if(adj) {
5157         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5158       }
5159     } // (!unconditional)
5160   } // if(ooo)
5161   else
5162   {
5163     // In-order execution (branch first)
5164     //if(likely[i]) printf("IOL\n");
5165     //else
5166     //printf("IOE\n");
5167     void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
5168     if(!unconditional&&!nop) {
5169       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5170       assert(s1l>=0);
5171       if((opcode[i]&0x2f)==4) // BEQ
5172       {
5173         if(s2l>=0) emit_cmp(s1l,s2l);
5174         else emit_test(s1l,s1l);
5175         nottaken=out;
5176         emit_jne(DJT_2);
5177       }
5178       if((opcode[i]&0x2f)==5) // BNE
5179       {
5180         if(s2l>=0) emit_cmp(s1l,s2l);
5181         else emit_test(s1l,s1l);
5182         nottaken=out;
5183         emit_jeq(DJT_2);
5184       }
5185       if((opcode[i]&0x2f)==6) // BLEZ
5186       {
5187         emit_cmpimm(s1l,1);
5188         nottaken=out;
5189         emit_jge(DJT_2);
5190       }
5191       if((opcode[i]&0x2f)==7) // BGTZ
5192       {
5193         emit_cmpimm(s1l,1);
5194         nottaken=out;
5195         emit_jl(DJT_2);
5196       }
5197     } // if(!unconditional)
5198     int adj;
5199     uint64_t ds_unneeded=branch_regs[i].u;
5200     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5201     ds_unneeded|=1;
5202     // branch taken
5203     if(!nop) {
5204       if(taken) set_jump_target(taken, out);
5205       assem_debug("1:\n");
5206       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5207       // load regs
5208       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5209       address_generation(i+1,&branch_regs[i],0);
5210       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5211       ds_assemble(i+1,&branch_regs[i]);
5212       cc=get_reg(branch_regs[i].regmap,CCREG);
5213       if(cc==-1) {
5214         emit_loadreg(CCREG,cc=HOST_CCREG);
5215         // CHECK: Is the following instruction (fall thru) allocated ok?
5216       }
5217       assert(cc==HOST_CCREG);
5218       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5219       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5220       assem_debug("cycle count (adj)\n");
5221       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5222       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5223       if(internal)
5224         assem_debug("branch: internal\n");
5225       else
5226         assem_debug("branch: external\n");
5227       if(internal&&is_ds[(ba[i]-start)>>2]) {
5228         ds_assemble_entry(i);
5229       }
5230       else {
5231         add_to_linker(out,ba[i],internal);
5232         emit_jmp(0);
5233       }
5234     }
5235     // branch not taken
5236     if(!unconditional) {
5237       if(nottaken1) set_jump_target(nottaken1, out);
5238       set_jump_target(nottaken, out);
5239       assem_debug("2:\n");
5240       if(!likely[i]) {
5241         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5242         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5243         address_generation(i+1,&branch_regs[i],0);
5244         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5245         ds_assemble(i+1,&branch_regs[i]);
5246       }
5247       cc=get_reg(branch_regs[i].regmap,CCREG);
5248       if(cc==-1&&!likely[i]) {
5249         // Cycle count isn't in a register, temporarily load it then write it out
5250         emit_loadreg(CCREG,HOST_CCREG);
5251         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5252         void *jaddr=out;
5253         emit_jns(0);
5254         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5255         emit_storereg(CCREG,HOST_CCREG);
5256       }
5257       else{
5258         cc=get_reg(i_regmap,CCREG);
5259         assert(cc==HOST_CCREG);
5260         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5261         void *jaddr=out;
5262         emit_jns(0);
5263         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5264       }
5265     }
5266   }
5267 }
5268
5269 static void sjump_assemble(int i,struct regstat *i_regs)
5270 {
5271   signed char *i_regmap=i_regs->regmap;
5272   int cc;
5273   int match;
5274   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5275   assem_debug("smatch=%d\n",match);
5276   int s1l;
5277   int unconditional=0,nevertaken=0;
5278   int invert=0;
5279   int internal=internal_branch(ba[i]);
5280   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
5281   if(!match) invert=1;
5282   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5283   if(i>(ba[i]-start)>>2) invert=1;
5284   #endif
5285   #ifdef __aarch64__
5286   invert=1; // because of near cond. branches
5287   #endif
5288
5289   //if(opcode2[i]>=0x10) return; // FIXME (BxxZAL)
5290   //assert(opcode2[i]<0x10||rs1[i]==0); // FIXME (BxxZAL)
5291
5292   if(ooo[i]) {
5293     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
5294   }
5295   else {
5296     s1l=get_reg(i_regmap,rs1[i]);
5297   }
5298   if(rs1[i]==0)
5299   {
5300     if(opcode2[i]&1) unconditional=1;
5301     else nevertaken=1;
5302     // These are never taken (r0 is never less than zero)
5303     //assert(opcode2[i]!=0);
5304     //assert(opcode2[i]!=2);
5305     //assert(opcode2[i]!=0x10);
5306     //assert(opcode2[i]!=0x12);
5307   }
5308
5309   if(ooo[i]) {
5310     // Out of order execution (delay slot first)
5311     //printf("OOOE\n");
5312     address_generation(i+1,i_regs,regs[i].regmap_entry);
5313     ds_assemble(i+1,i_regs);
5314     int adj;
5315     uint64_t bc_unneeded=branch_regs[i].u;
5316     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5317     bc_unneeded|=1;
5318     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
5319     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs1[i]);
5320     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5321     if(rt1[i]==31) {
5322       int rt,return_address;
5323       rt=get_reg(branch_regs[i].regmap,31);
5324       assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5325       if(rt>=0) {
5326         // Save the PC even if the branch is not taken
5327         return_address=start+i*4+8;
5328         emit_movimm(return_address,rt); // PC into link register
5329         #ifdef IMM_PREFETCH
5330         if(!nevertaken) emit_prefetch(hash_table_get(return_address));
5331         #endif
5332       }
5333     }
5334     cc=get_reg(branch_regs[i].regmap,CCREG);
5335     assert(cc==HOST_CCREG);
5336     if(unconditional)
5337       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5338     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
5339     assem_debug("cycle count (adj)\n");
5340     if(unconditional) {
5341       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5342       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
5343         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5344         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5345         if(internal)
5346           assem_debug("branch: internal\n");
5347         else
5348           assem_debug("branch: external\n");
5349         if(internal&&is_ds[(ba[i]-start)>>2]) {
5350           ds_assemble_entry(i);
5351         }
5352         else {
5353           add_to_linker(out,ba[i],internal);
5354           emit_jmp(0);
5355         }
5356         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5357         if(((u_int)out)&7) emit_addnop(0);
5358         #endif
5359       }
5360     }
5361     else if(nevertaken) {
5362       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5363       void *jaddr=out;
5364       emit_jns(0);
5365       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5366     }
5367     else {
5368       void *nottaken = NULL;
5369       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5370       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5371       {
5372         assert(s1l>=0);
5373         if((opcode2[i]&0xf)==0) // BLTZ/BLTZAL
5374         {
5375           emit_test(s1l,s1l);
5376           if(invert){
5377             nottaken=out;
5378             emit_jns(DJT_1);
5379           }else{
5380             add_to_linker(out,ba[i],internal);
5381             emit_js(0);
5382           }
5383         }
5384         if((opcode2[i]&0xf)==1) // BGEZ/BLTZAL
5385         {
5386           emit_test(s1l,s1l);
5387           if(invert){
5388             nottaken=out;
5389             emit_js(DJT_1);
5390           }else{
5391             add_to_linker(out,ba[i],internal);
5392             emit_jns(0);
5393           }
5394         }
5395       }
5396
5397       if(invert) {
5398         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5399         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5400           if(adj) {
5401             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5402             add_to_linker(out,ba[i],internal);
5403           }else{
5404             emit_addnop(13);
5405             add_to_linker(out,ba[i],internal*2);
5406           }
5407           emit_jmp(0);
5408         }else
5409         #endif
5410         {
5411           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5412           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5413           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5414           if(internal)
5415             assem_debug("branch: internal\n");
5416           else
5417             assem_debug("branch: external\n");
5418           if(internal&&is_ds[(ba[i]-start)>>2]) {
5419             ds_assemble_entry(i);
5420           }
5421           else {
5422             add_to_linker(out,ba[i],internal);
5423             emit_jmp(0);
5424           }
5425         }
5426         set_jump_target(nottaken, out);
5427       }
5428
5429       if(adj) {
5430         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5431       }
5432     } // (!unconditional)
5433   } // if(ooo)
5434   else
5435   {
5436     // In-order execution (branch first)
5437     //printf("IOE\n");
5438     void *nottaken = NULL;
5439     if(rt1[i]==31) {
5440       int rt,return_address;
5441       rt=get_reg(branch_regs[i].regmap,31);
5442       if(rt>=0) {
5443         // Save the PC even if the branch is not taken
5444         return_address=start+i*4+8;
5445         emit_movimm(return_address,rt); // PC into link register
5446         #ifdef IMM_PREFETCH
5447         emit_prefetch(hash_table_get(return_address));
5448         #endif
5449       }
5450     }
5451     if(!unconditional) {
5452       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5453         assert(s1l>=0);
5454         if((opcode2[i]&0x0d)==0) // BLTZ/BLTZL/BLTZAL/BLTZALL
5455         {
5456           emit_test(s1l,s1l);
5457           nottaken=out;
5458           emit_jns(DJT_1);
5459         }
5460         if((opcode2[i]&0x0d)==1) // BGEZ/BGEZL/BGEZAL/BGEZALL
5461         {
5462           emit_test(s1l,s1l);
5463           nottaken=out;
5464           emit_js(DJT_1);
5465         }
5466     } // if(!unconditional)
5467     int adj;
5468     uint64_t ds_unneeded=branch_regs[i].u;
5469     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5470     ds_unneeded|=1;
5471     // branch taken
5472     if(!nevertaken) {
5473       //assem_debug("1:\n");
5474       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5475       // load regs
5476       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5477       address_generation(i+1,&branch_regs[i],0);
5478       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5479       ds_assemble(i+1,&branch_regs[i]);
5480       cc=get_reg(branch_regs[i].regmap,CCREG);
5481       if(cc==-1) {
5482         emit_loadreg(CCREG,cc=HOST_CCREG);
5483         // CHECK: Is the following instruction (fall thru) allocated ok?
5484       }
5485       assert(cc==HOST_CCREG);
5486       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5487       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5488       assem_debug("cycle count (adj)\n");
5489       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5490       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5491       if(internal)
5492         assem_debug("branch: internal\n");
5493       else
5494         assem_debug("branch: external\n");
5495       if(internal&&is_ds[(ba[i]-start)>>2]) {
5496         ds_assemble_entry(i);
5497       }
5498       else {
5499         add_to_linker(out,ba[i],internal);
5500         emit_jmp(0);
5501       }
5502     }
5503     // branch not taken
5504     if(!unconditional) {
5505       set_jump_target(nottaken, out);
5506       assem_debug("1:\n");
5507       if(!likely[i]) {
5508         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5509         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5510         address_generation(i+1,&branch_regs[i],0);
5511         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5512         ds_assemble(i+1,&branch_regs[i]);
5513       }
5514       cc=get_reg(branch_regs[i].regmap,CCREG);
5515       if(cc==-1&&!likely[i]) {
5516         // Cycle count isn't in a register, temporarily load it then write it out
5517         emit_loadreg(CCREG,HOST_CCREG);
5518         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5519         void *jaddr=out;
5520         emit_jns(0);
5521         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5522         emit_storereg(CCREG,HOST_CCREG);
5523       }
5524       else{
5525         cc=get_reg(i_regmap,CCREG);
5526         assert(cc==HOST_CCREG);
5527         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5528         void *jaddr=out;
5529         emit_jns(0);
5530         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5531       }
5532     }
5533   }
5534 }
5535
5536 static void pagespan_assemble(int i,struct regstat *i_regs)
5537 {
5538   int s1l=get_reg(i_regs->regmap,rs1[i]);
5539   int s2l=get_reg(i_regs->regmap,rs2[i]);
5540   void *taken = NULL;
5541   void *nottaken = NULL;
5542   int unconditional=0;
5543   if(rs1[i]==0)
5544   {
5545     s1l=s2l;
5546     s2l=-1;
5547   }
5548   else if(rs2[i]==0)
5549   {
5550     s2l=-1;
5551   }
5552   int hr=0;
5553   int addr=-1,alt=-1,ntaddr=-1;
5554   if(i_regs->regmap[HOST_BTREG]<0) {addr=HOST_BTREG;}
5555   else {
5556     while(hr<HOST_REGS)
5557     {
5558       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
5559          (i_regs->regmap[hr]&63)!=rs1[i] &&
5560          (i_regs->regmap[hr]&63)!=rs2[i] )
5561       {
5562         addr=hr++;break;
5563       }
5564       hr++;
5565     }
5566   }
5567   while(hr<HOST_REGS)
5568   {
5569     if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5570        (i_regs->regmap[hr]&63)!=rs1[i] &&
5571        (i_regs->regmap[hr]&63)!=rs2[i] )
5572     {
5573       alt=hr++;break;
5574     }
5575     hr++;
5576   }
5577   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
5578   {
5579     while(hr<HOST_REGS)
5580     {
5581       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5582          (i_regs->regmap[hr]&63)!=rs1[i] &&
5583          (i_regs->regmap[hr]&63)!=rs2[i] )
5584       {
5585         ntaddr=hr;break;
5586       }
5587       hr++;
5588     }
5589   }
5590   assert(hr<HOST_REGS);
5591   if((opcode[i]&0x2e)==4||opcode[i]==0x11) { // BEQ/BNE/BEQL/BNEL/BC1
5592     load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
5593   }
5594   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5595   if(opcode[i]==2) // J
5596   {
5597     unconditional=1;
5598   }
5599   if(opcode[i]==3) // JAL
5600   {
5601     // TODO: mini_ht
5602     int rt=get_reg(i_regs->regmap,31);
5603     emit_movimm(start+i*4+8,rt);
5604     unconditional=1;
5605   }
5606   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
5607   {
5608     emit_mov(s1l,addr);
5609     if(opcode2[i]==9) // JALR
5610     {
5611       int rt=get_reg(i_regs->regmap,rt1[i]);
5612       emit_movimm(start+i*4+8,rt);
5613     }
5614   }
5615   if((opcode[i]&0x3f)==4) // BEQ
5616   {
5617     if(rs1[i]==rs2[i])
5618     {
5619       unconditional=1;
5620     }
5621     else
5622     #ifdef HAVE_CMOV_IMM
5623     if(1) {
5624       if(s2l>=0) emit_cmp(s1l,s2l);
5625       else emit_test(s1l,s1l);
5626       emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
5627     }
5628     else
5629     #endif
5630     {
5631       assert(s1l>=0);
5632       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5633       if(s2l>=0) emit_cmp(s1l,s2l);
5634       else emit_test(s1l,s1l);
5635       emit_cmovne_reg(alt,addr);
5636     }
5637   }
5638   if((opcode[i]&0x3f)==5) // BNE
5639   {
5640     #ifdef HAVE_CMOV_IMM
5641     if(s2l>=0) emit_cmp(s1l,s2l);
5642     else emit_test(s1l,s1l);
5643     emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
5644     #else
5645     assert(s1l>=0);
5646     emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
5647     if(s2l>=0) emit_cmp(s1l,s2l);
5648     else emit_test(s1l,s1l);
5649     emit_cmovne_reg(alt,addr);
5650     #endif
5651   }
5652   if((opcode[i]&0x3f)==0x14) // BEQL
5653   {
5654     if(s2l>=0) emit_cmp(s1l,s2l);
5655     else emit_test(s1l,s1l);
5656     if(nottaken) set_jump_target(nottaken, out);
5657     nottaken=out;
5658     emit_jne(0);
5659   }
5660   if((opcode[i]&0x3f)==0x15) // BNEL
5661   {
5662     if(s2l>=0) emit_cmp(s1l,s2l);
5663     else emit_test(s1l,s1l);
5664     nottaken=out;
5665     emit_jeq(0);
5666     if(taken) set_jump_target(taken, out);
5667   }
5668   if((opcode[i]&0x3f)==6) // BLEZ
5669   {
5670     emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5671     emit_cmpimm(s1l,1);
5672     emit_cmovl_reg(alt,addr);
5673   }
5674   if((opcode[i]&0x3f)==7) // BGTZ
5675   {
5676     emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
5677     emit_cmpimm(s1l,1);
5678     emit_cmovl_reg(ntaddr,addr);
5679   }
5680   if((opcode[i]&0x3f)==0x16) // BLEZL
5681   {
5682     assert((opcode[i]&0x3f)!=0x16);
5683   }
5684   if((opcode[i]&0x3f)==0x17) // BGTZL
5685   {
5686     assert((opcode[i]&0x3f)!=0x17);
5687   }
5688   assert(opcode[i]!=1); // BLTZ/BGEZ
5689
5690   //FIXME: Check CSREG
5691   if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
5692     if((source[i]&0x30000)==0) // BC1F
5693     {
5694       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5695       emit_testimm(s1l,0x800000);
5696       emit_cmovne_reg(alt,addr);
5697     }
5698     if((source[i]&0x30000)==0x10000) // BC1T
5699     {
5700       emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5701       emit_testimm(s1l,0x800000);
5702       emit_cmovne_reg(alt,addr);
5703     }
5704     if((source[i]&0x30000)==0x20000) // BC1FL
5705     {
5706       emit_testimm(s1l,0x800000);
5707       nottaken=out;
5708       emit_jne(0);
5709     }
5710     if((source[i]&0x30000)==0x30000) // BC1TL
5711     {
5712       emit_testimm(s1l,0x800000);
5713       nottaken=out;
5714       emit_jeq(0);
5715     }
5716   }
5717
5718   assert(i_regs->regmap[HOST_CCREG]==CCREG);
5719   wb_dirtys(regs[i].regmap,regs[i].dirty);
5720   if(likely[i]||unconditional)
5721   {
5722     emit_movimm(ba[i],HOST_BTREG);
5723   }
5724   else if(addr!=HOST_BTREG)
5725   {
5726     emit_mov(addr,HOST_BTREG);
5727   }
5728   void *branch_addr=out;
5729   emit_jmp(0);
5730   int target_addr=start+i*4+5;
5731   void *stub=out;
5732   void *compiled_target_addr=check_addr(target_addr);
5733   emit_extjump_ds(branch_addr, target_addr);
5734   if(compiled_target_addr) {
5735     set_jump_target(branch_addr, compiled_target_addr);
5736     add_link(target_addr,stub);
5737   }
5738   else set_jump_target(branch_addr, stub);
5739   if(likely[i]) {
5740     // Not-taken path
5741     set_jump_target(nottaken, out);
5742     wb_dirtys(regs[i].regmap,regs[i].dirty);
5743     void *branch_addr=out;
5744     emit_jmp(0);
5745     int target_addr=start+i*4+8;
5746     void *stub=out;
5747     void *compiled_target_addr=check_addr(target_addr);
5748     emit_extjump_ds(branch_addr, target_addr);
5749     if(compiled_target_addr) {
5750       set_jump_target(branch_addr, compiled_target_addr);
5751       add_link(target_addr,stub);
5752     }
5753     else set_jump_target(branch_addr, stub);
5754   }
5755 }
5756
5757 // Assemble the delay slot for the above
5758 static void pagespan_ds()
5759 {
5760   assem_debug("initial delay slot:\n");
5761   u_int vaddr=start+1;
5762   u_int page=get_page(vaddr);
5763   u_int vpage=get_vpage(vaddr);
5764   ll_add(jump_dirty+vpage,vaddr,(void *)out);
5765   do_dirty_stub_ds();
5766   ll_add(jump_in+page,vaddr,(void *)out);
5767   assert(regs[0].regmap_entry[HOST_CCREG]==CCREG);
5768   if(regs[0].regmap[HOST_CCREG]!=CCREG)
5769     wb_register(CCREG,regs[0].regmap_entry,regs[0].wasdirty);
5770   if(regs[0].regmap[HOST_BTREG]!=BTREG)
5771     emit_writeword(HOST_BTREG,&branch_target);
5772   load_regs(regs[0].regmap_entry,regs[0].regmap,rs1[0],rs2[0]);
5773   address_generation(0,&regs[0],regs[0].regmap_entry);
5774   if(itype[0]==STORE||itype[0]==STORELR||(opcode[0]&0x3b)==0x39||(opcode[0]&0x3b)==0x3a)
5775     load_regs(regs[0].regmap_entry,regs[0].regmap,INVCP,INVCP);
5776   is_delayslot=0;
5777   switch(itype[0]) {
5778     case ALU:
5779       alu_assemble(0,&regs[0]);break;
5780     case IMM16:
5781       imm16_assemble(0,&regs[0]);break;
5782     case SHIFT:
5783       shift_assemble(0,&regs[0]);break;
5784     case SHIFTIMM:
5785       shiftimm_assemble(0,&regs[0]);break;
5786     case LOAD:
5787       load_assemble(0,&regs[0]);break;
5788     case LOADLR:
5789       loadlr_assemble(0,&regs[0]);break;
5790     case STORE:
5791       store_assemble(0,&regs[0]);break;
5792     case STORELR:
5793       storelr_assemble(0,&regs[0]);break;
5794     case COP0:
5795       cop0_assemble(0,&regs[0]);break;
5796     case COP1:
5797       cop1_assemble(0,&regs[0]);break;
5798     case C1LS:
5799       c1ls_assemble(0,&regs[0]);break;
5800     case COP2:
5801       cop2_assemble(0,&regs[0]);break;
5802     case C2LS:
5803       c2ls_assemble(0,&regs[0]);break;
5804     case C2OP:
5805       c2op_assemble(0,&regs[0]);break;
5806     case MULTDIV:
5807       multdiv_assemble(0,&regs[0]);break;
5808     case MOV:
5809       mov_assemble(0,&regs[0]);break;
5810     case SYSCALL:
5811     case HLECALL:
5812     case INTCALL:
5813     case SPAN:
5814     case UJUMP:
5815     case RJUMP:
5816     case CJUMP:
5817     case SJUMP:
5818       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
5819   }
5820   int btaddr=get_reg(regs[0].regmap,BTREG);
5821   if(btaddr<0) {
5822     btaddr=get_reg(regs[0].regmap,-1);
5823     emit_readword(&branch_target,btaddr);
5824   }
5825   assert(btaddr!=HOST_CCREG);
5826   if(regs[0].regmap[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
5827 #ifdef HOST_IMM8
5828   host_tempreg_acquire();
5829   emit_movimm(start+4,HOST_TEMPREG);
5830   emit_cmp(btaddr,HOST_TEMPREG);
5831   host_tempreg_release();
5832 #else
5833   emit_cmpimm(btaddr,start+4);
5834 #endif
5835   void *branch = out;
5836   emit_jeq(0);
5837   store_regs_bt(regs[0].regmap,regs[0].dirty,-1);
5838   do_jump_vaddr(btaddr);
5839   set_jump_target(branch, out);
5840   store_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5841   load_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5842 }
5843
5844 // Basic liveness analysis for MIPS registers
5845 void unneeded_registers(int istart,int iend,int r)
5846 {
5847   int i;
5848   uint64_t u,gte_u,b,gte_b;
5849   uint64_t temp_u,temp_gte_u=0;
5850   uint64_t gte_u_unknown=0;
5851   if (HACK_ENABLED(NDHACK_GTE_UNNEEDED))
5852     gte_u_unknown=~0ll;
5853   if(iend==slen-1) {
5854     u=1;
5855     gte_u=gte_u_unknown;
5856   }else{
5857     //u=unneeded_reg[iend+1];
5858     u=1;
5859     gte_u=gte_unneeded[iend+1];
5860   }
5861
5862   for (i=iend;i>=istart;i--)
5863   {
5864     //printf("unneeded registers i=%d (%d,%d) r=%d\n",i,istart,iend,r);
5865     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5866     {
5867       // If subroutine call, flag return address as a possible branch target
5868       if(rt1[i]==31 && i<slen-2) bt[i+2]=1;
5869
5870       if(ba[i]<start || ba[i]>=(start+slen*4))
5871       {
5872         // Branch out of this block, flush all regs
5873         u=1;
5874         gte_u=gte_u_unknown;
5875         branch_unneeded_reg[i]=u;
5876         // Merge in delay slot
5877         u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5878         u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5879         u|=1;
5880         gte_u|=gte_rt[i+1];
5881         gte_u&=~gte_rs[i+1];
5882         // If branch is "likely" (and conditional)
5883         // then we skip the delay slot on the fall-thru path
5884         if(likely[i]) {
5885           if(i<slen-1) {
5886             u&=unneeded_reg[i+2];
5887             gte_u&=gte_unneeded[i+2];
5888           }
5889           else
5890           {
5891             u=1;
5892             gte_u=gte_u_unknown;
5893           }
5894         }
5895       }
5896       else
5897       {
5898         // Internal branch, flag target
5899         bt[(ba[i]-start)>>2]=1;
5900         if(ba[i]<=start+i*4) {
5901           // Backward branch
5902           if(is_ujump(i))
5903           {
5904             // Unconditional branch
5905             temp_u=1;
5906             temp_gte_u=0;
5907           } else {
5908             // Conditional branch (not taken case)
5909             temp_u=unneeded_reg[i+2];
5910             temp_gte_u&=gte_unneeded[i+2];
5911           }
5912           // Merge in delay slot
5913           temp_u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5914           temp_u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5915           temp_u|=1;
5916           temp_gte_u|=gte_rt[i+1];
5917           temp_gte_u&=~gte_rs[i+1];
5918           // If branch is "likely" (and conditional)
5919           // then we skip the delay slot on the fall-thru path
5920           if(likely[i]) {
5921             if(i<slen-1) {
5922               temp_u&=unneeded_reg[i+2];
5923               temp_gte_u&=gte_unneeded[i+2];
5924             }
5925             else
5926             {
5927               temp_u=1;
5928               temp_gte_u=gte_u_unknown;
5929             }
5930           }
5931           temp_u|=(1LL<<rt1[i])|(1LL<<rt2[i]);
5932           temp_u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5933           temp_u|=1;
5934           temp_gte_u|=gte_rt[i];
5935           temp_gte_u&=~gte_rs[i];
5936           unneeded_reg[i]=temp_u;
5937           gte_unneeded[i]=temp_gte_u;
5938           // Only go three levels deep.  This recursion can take an
5939           // excessive amount of time if there are a lot of nested loops.
5940           if(r<2) {
5941             unneeded_registers((ba[i]-start)>>2,i-1,r+1);
5942           }else{
5943             unneeded_reg[(ba[i]-start)>>2]=1;
5944             gte_unneeded[(ba[i]-start)>>2]=gte_u_unknown;
5945           }
5946         } /*else*/ if(1) {
5947           if (is_ujump(i))
5948           {
5949             // Unconditional branch
5950             u=unneeded_reg[(ba[i]-start)>>2];
5951             gte_u=gte_unneeded[(ba[i]-start)>>2];
5952             branch_unneeded_reg[i]=u;
5953             // Merge in delay slot
5954             u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5955             u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5956             u|=1;
5957             gte_u|=gte_rt[i+1];
5958             gte_u&=~gte_rs[i+1];
5959           } else {
5960             // Conditional branch
5961             b=unneeded_reg[(ba[i]-start)>>2];
5962             gte_b=gte_unneeded[(ba[i]-start)>>2];
5963             branch_unneeded_reg[i]=b;
5964             // Branch delay slot
5965             b|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5966             b&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5967             b|=1;
5968             gte_b|=gte_rt[i+1];
5969             gte_b&=~gte_rs[i+1];
5970             // If branch is "likely" then we skip the
5971             // delay slot on the fall-thru path
5972             if(likely[i]) {
5973               u=b;
5974               gte_u=gte_b;
5975               if(i<slen-1) {
5976                 u&=unneeded_reg[i+2];
5977                 gte_u&=gte_unneeded[i+2];
5978               }
5979             } else {
5980               u&=b;
5981               gte_u&=gte_b;
5982             }
5983             if(i<slen-1) {
5984               branch_unneeded_reg[i]&=unneeded_reg[i+2];
5985             } else {
5986               branch_unneeded_reg[i]=1;
5987             }
5988           }
5989         }
5990       }
5991     }
5992     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5993     {
5994       // SYSCALL instruction (software interrupt)
5995       u=1;
5996     }
5997     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5998     {
5999       // ERET instruction (return from interrupt)
6000       u=1;
6001     }
6002     //u=1; // DEBUG
6003     // Written registers are unneeded
6004     u|=1LL<<rt1[i];
6005     u|=1LL<<rt2[i];
6006     gte_u|=gte_rt[i];
6007     // Accessed registers are needed
6008     u&=~(1LL<<rs1[i]);
6009     u&=~(1LL<<rs2[i]);
6010     gte_u&=~gte_rs[i];
6011     if(gte_rs[i]&&rt1[i]&&(unneeded_reg[i+1]&(1ll<<rt1[i])))
6012       gte_u|=gte_rs[i]&gte_unneeded[i+1]; // MFC2/CFC2 to dead register, unneeded
6013     // Source-target dependencies
6014     // R0 is always unneeded
6015     u|=1;
6016     // Save it
6017     unneeded_reg[i]=u;
6018     gte_unneeded[i]=gte_u;
6019     /*
6020     printf("ur (%d,%d) %x: ",istart,iend,start+i*4);
6021     printf("U:");
6022     int r;
6023     for(r=1;r<=CCREG;r++) {
6024       if((unneeded_reg[i]>>r)&1) {
6025         if(r==HIREG) printf(" HI");
6026         else if(r==LOREG) printf(" LO");
6027         else printf(" r%d",r);
6028       }
6029     }
6030     printf("\n");
6031     */
6032   }
6033 }
6034
6035 // Write back dirty registers as soon as we will no longer modify them,
6036 // so that we don't end up with lots of writes at the branches.
6037 void clean_registers(int istart,int iend,int wr)
6038 {
6039   int i;
6040   int r;
6041   u_int will_dirty_i,will_dirty_next,temp_will_dirty;
6042   u_int wont_dirty_i,wont_dirty_next,temp_wont_dirty;
6043   if(iend==slen-1) {
6044     will_dirty_i=will_dirty_next=0;
6045     wont_dirty_i=wont_dirty_next=0;
6046   }else{
6047     will_dirty_i=will_dirty_next=will_dirty[iend+1];
6048     wont_dirty_i=wont_dirty_next=wont_dirty[iend+1];
6049   }
6050   for (i=iend;i>=istart;i--)
6051   {
6052     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
6053     {
6054       if(ba[i]<start || ba[i]>=(start+slen*4))
6055       {
6056         // Branch out of this block, flush all regs
6057         if (is_ujump(i))
6058         {
6059           // Unconditional branch
6060           will_dirty_i=0;
6061           wont_dirty_i=0;
6062           // Merge in delay slot (will dirty)
6063           for(r=0;r<HOST_REGS;r++) {
6064             if(r!=EXCLUDE_REG) {
6065               if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6066               if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6067               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6068               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6069               if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6070               if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6071               if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6072               if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6073               if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6074               if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6075               if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6076               if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6077               if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6078               if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6079             }
6080           }
6081         }
6082         else
6083         {
6084           // Conditional branch
6085           will_dirty_i=0;
6086           wont_dirty_i=wont_dirty_next;
6087           // Merge in delay slot (will dirty)
6088           for(r=0;r<HOST_REGS;r++) {
6089             if(r!=EXCLUDE_REG) {
6090               if(!likely[i]) {
6091                 // Might not dirty if likely branch is not taken
6092                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6093                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6094                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6095                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6096                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6097                 if(branch_regs[i].regmap[r]==0) will_dirty_i&=~(1<<r);
6098                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6099                 //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6100                 //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6101                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6102                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6103                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6104                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6105                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6106               }
6107             }
6108           }
6109         }
6110         // Merge in delay slot (wont dirty)
6111         for(r=0;r<HOST_REGS;r++) {
6112           if(r!=EXCLUDE_REG) {
6113             if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6114             if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6115             if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6116             if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6117             if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6118             if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6119             if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6120             if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6121             if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6122             if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6123           }
6124         }
6125         if(wr) {
6126           #ifndef DESTRUCTIVE_WRITEBACK
6127           branch_regs[i].dirty&=wont_dirty_i;
6128           #endif
6129           branch_regs[i].dirty|=will_dirty_i;
6130         }
6131       }
6132       else
6133       {
6134         // Internal branch
6135         if(ba[i]<=start+i*4) {
6136           // Backward branch
6137           if (is_ujump(i))
6138           {
6139             // Unconditional branch
6140             temp_will_dirty=0;
6141             temp_wont_dirty=0;
6142             // Merge in delay slot (will dirty)
6143             for(r=0;r<HOST_REGS;r++) {
6144               if(r!=EXCLUDE_REG) {
6145                 if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6146                 if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6147                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6148                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6149                 if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6150                 if(branch_regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6151                 if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6152                 if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6153                 if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6154                 if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6155                 if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6156                 if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6157                 if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6158                 if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6159               }
6160             }
6161           } else {
6162             // Conditional branch (not taken case)
6163             temp_will_dirty=will_dirty_next;
6164             temp_wont_dirty=wont_dirty_next;
6165             // Merge in delay slot (will dirty)
6166             for(r=0;r<HOST_REGS;r++) {
6167               if(r!=EXCLUDE_REG) {
6168                 if(!likely[i]) {
6169                   // Will not dirty if likely branch is not taken
6170                   if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6171                   if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6172                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6173                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6174                   if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6175                   if(branch_regs[i].regmap[r]==0) temp_will_dirty&=~(1<<r);
6176                   if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6177                   //if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6178                   //if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6179                   if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6180                   if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6181                   if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6182                   if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6183                   if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6184                 }
6185               }
6186             }
6187           }
6188           // Merge in delay slot (wont dirty)
6189           for(r=0;r<HOST_REGS;r++) {
6190             if(r!=EXCLUDE_REG) {
6191               if((regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
6192               if((regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
6193               if((regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
6194               if((regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
6195               if(regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
6196               if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
6197               if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
6198               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
6199               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
6200               if(branch_regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
6201             }
6202           }
6203           // Deal with changed mappings
6204           if(i<iend) {
6205             for(r=0;r<HOST_REGS;r++) {
6206               if(r!=EXCLUDE_REG) {
6207                 if(regs[i].regmap[r]!=regmap_pre[i][r]) {
6208                   temp_will_dirty&=~(1<<r);
6209                   temp_wont_dirty&=~(1<<r);
6210                   if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6211                     temp_will_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6212                     temp_wont_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6213                   } else {
6214                     temp_will_dirty|=1<<r;
6215                     temp_wont_dirty|=1<<r;
6216                   }
6217                 }
6218               }
6219             }
6220           }
6221           if(wr) {
6222             will_dirty[i]=temp_will_dirty;
6223             wont_dirty[i]=temp_wont_dirty;
6224             clean_registers((ba[i]-start)>>2,i-1,0);
6225           }else{
6226             // Limit recursion.  It can take an excessive amount
6227             // of time if there are a lot of nested loops.
6228             will_dirty[(ba[i]-start)>>2]=0;
6229             wont_dirty[(ba[i]-start)>>2]=-1;
6230           }
6231         }
6232         /*else*/ if(1)
6233         {
6234           if (is_ujump(i))
6235           {
6236             // Unconditional branch
6237             will_dirty_i=0;
6238             wont_dirty_i=0;
6239           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
6240             for(r=0;r<HOST_REGS;r++) {
6241               if(r!=EXCLUDE_REG) {
6242                 if(branch_regs[i].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6243                   will_dirty_i|=will_dirty[(ba[i]-start)>>2]&(1<<r);
6244                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6245                 }
6246                 if(branch_regs[i].regmap[r]>=0) {
6247                   will_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
6248                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
6249                 }
6250               }
6251             }
6252           //}
6253             // Merge in delay slot
6254             for(r=0;r<HOST_REGS;r++) {
6255               if(r!=EXCLUDE_REG) {
6256                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6257                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6258                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6259                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6260                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6261                 if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6262                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6263                 if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6264                 if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6265                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6266                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6267                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6268                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6269                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6270               }
6271             }
6272           } else {
6273             // Conditional branch
6274             will_dirty_i=will_dirty_next;
6275             wont_dirty_i=wont_dirty_next;
6276           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
6277             for(r=0;r<HOST_REGS;r++) {
6278               if(r!=EXCLUDE_REG) {
6279                 signed char target_reg=branch_regs[i].regmap[r];
6280                 if(target_reg==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6281                   will_dirty_i&=will_dirty[(ba[i]-start)>>2]&(1<<r);
6282                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6283                 }
6284                 else if(target_reg>=0) {
6285                   will_dirty_i&=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
6286                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
6287                 }
6288                 // Treat delay slot as part of branch too
6289                 /*if(regs[i+1].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6290                   will_dirty[i+1]&=will_dirty[(ba[i]-start)>>2]&(1<<r);
6291                   wont_dirty[i+1]|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6292                 }
6293                 else
6294                 {
6295                   will_dirty[i+1]&=~(1<<r);
6296                 }*/
6297               }
6298             }
6299           //}
6300             // Merge in delay slot
6301             for(r=0;r<HOST_REGS;r++) {
6302               if(r!=EXCLUDE_REG) {
6303                 if(!likely[i]) {
6304                   // Might not dirty if likely branch is not taken
6305                   if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6306                   if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6307                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6308                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6309                   if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6310                   if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6311                   if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6312                   //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6313                   //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6314                   if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6315                   if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6316                   if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6317                   if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6318                   if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6319                 }
6320               }
6321             }
6322           }
6323           // Merge in delay slot (won't dirty)
6324           for(r=0;r<HOST_REGS;r++) {
6325             if(r!=EXCLUDE_REG) {
6326               if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6327               if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6328               if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6329               if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6330               if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6331               if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6332               if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6333               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6334               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6335               if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6336             }
6337           }
6338           if(wr) {
6339             #ifndef DESTRUCTIVE_WRITEBACK
6340             branch_regs[i].dirty&=wont_dirty_i;
6341             #endif
6342             branch_regs[i].dirty|=will_dirty_i;
6343           }
6344         }
6345       }
6346     }
6347     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
6348     {
6349       // SYSCALL instruction (software interrupt)
6350       will_dirty_i=0;
6351       wont_dirty_i=0;
6352     }
6353     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
6354     {
6355       // ERET instruction (return from interrupt)
6356       will_dirty_i=0;
6357       wont_dirty_i=0;
6358     }
6359     will_dirty_next=will_dirty_i;
6360     wont_dirty_next=wont_dirty_i;
6361     for(r=0;r<HOST_REGS;r++) {
6362       if(r!=EXCLUDE_REG) {
6363         if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6364         if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6365         if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6366         if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6367         if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6368         if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6369         if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6370         if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6371         if(i>istart) {
6372           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP)
6373           {
6374             // Don't store a register immediately after writing it,
6375             // may prevent dual-issue.
6376             if((regs[i].regmap[r]&63)==rt1[i-1]) wont_dirty_i|=1<<r;
6377             if((regs[i].regmap[r]&63)==rt2[i-1]) wont_dirty_i|=1<<r;
6378           }
6379         }
6380       }
6381     }
6382     // Save it
6383     will_dirty[i]=will_dirty_i;
6384     wont_dirty[i]=wont_dirty_i;
6385     // Mark registers that won't be dirtied as not dirty
6386     if(wr) {
6387       /*printf("wr (%d,%d) %x will:",istart,iend,start+i*4);
6388       for(r=0;r<HOST_REGS;r++) {
6389         if((will_dirty_i>>r)&1) {
6390           printf(" r%d",r);
6391         }
6392       }
6393       printf("\n");*/
6394
6395       //if(i==istart||(itype[i-1]!=RJUMP&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP)) {
6396         regs[i].dirty|=will_dirty_i;
6397         #ifndef DESTRUCTIVE_WRITEBACK
6398         regs[i].dirty&=wont_dirty_i;
6399         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
6400         {
6401           if (i < iend-1 && !is_ujump(i)) {
6402             for(r=0;r<HOST_REGS;r++) {
6403               if(r!=EXCLUDE_REG) {
6404                 if(regs[i].regmap[r]==regmap_pre[i+2][r]) {
6405                   regs[i+2].wasdirty&=wont_dirty_i|~(1<<r);
6406                 }else {/*printf("i: %x (%d) mismatch(+2): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6407               }
6408             }
6409           }
6410         }
6411         else
6412         {
6413           if(i<iend) {
6414             for(r=0;r<HOST_REGS;r++) {
6415               if(r!=EXCLUDE_REG) {
6416                 if(regs[i].regmap[r]==regmap_pre[i+1][r]) {
6417                   regs[i+1].wasdirty&=wont_dirty_i|~(1<<r);
6418                 }else {/*printf("i: %x (%d) mismatch(+1): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6419               }
6420             }
6421           }
6422         }
6423         #endif
6424       //}
6425     }
6426     // Deal with changed mappings
6427     temp_will_dirty=will_dirty_i;
6428     temp_wont_dirty=wont_dirty_i;
6429     for(r=0;r<HOST_REGS;r++) {
6430       if(r!=EXCLUDE_REG) {
6431         int nr;
6432         if(regs[i].regmap[r]==regmap_pre[i][r]) {
6433           if(wr) {
6434             #ifndef DESTRUCTIVE_WRITEBACK
6435             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6436             #endif
6437             regs[i].wasdirty|=will_dirty_i&(1<<r);
6438           }
6439         }
6440         else if(regmap_pre[i][r]>=0&&(nr=get_reg(regs[i].regmap,regmap_pre[i][r]))>=0) {
6441           // Register moved to a different register
6442           will_dirty_i&=~(1<<r);
6443           wont_dirty_i&=~(1<<r);
6444           will_dirty_i|=((temp_will_dirty>>nr)&1)<<r;
6445           wont_dirty_i|=((temp_wont_dirty>>nr)&1)<<r;
6446           if(wr) {
6447             #ifndef DESTRUCTIVE_WRITEBACK
6448             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6449             #endif
6450             regs[i].wasdirty|=will_dirty_i&(1<<r);
6451           }
6452         }
6453         else {
6454           will_dirty_i&=~(1<<r);
6455           wont_dirty_i&=~(1<<r);
6456           if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6457             will_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6458             wont_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6459           } else {
6460             wont_dirty_i|=1<<r;
6461             /*printf("i: %x (%d) mismatch: %d\n",start+i*4,i,r);assert(!((will_dirty>>r)&1));*/
6462           }
6463         }
6464       }
6465     }
6466   }
6467 }
6468
6469 #ifdef DISASM
6470   /* disassembly */
6471 void disassemble_inst(int i)
6472 {
6473     if (bt[i]) printf("*"); else printf(" ");
6474     switch(itype[i]) {
6475       case UJUMP:
6476         printf (" %x: %s %8x\n",start+i*4,insn[i],ba[i]);break;
6477       case CJUMP:
6478         printf (" %x: %s r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],i?start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14):*ba);break;
6479       case SJUMP:
6480         printf (" %x: %s r%d,%8x\n",start+i*4,insn[i],rs1[i],start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14));break;
6481       case RJUMP:
6482         if (opcode[i]==0x9&&rt1[i]!=31)
6483           printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i]);
6484         else
6485           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6486         break;
6487       case SPAN:
6488         printf (" %x: %s (pagespan) r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],ba[i]);break;
6489       case IMM16:
6490         if(opcode[i]==0xf) //LUI
6491           printf (" %x: %s r%d,%4x0000\n",start+i*4,insn[i],rt1[i],imm[i]&0xffff);
6492         else
6493           printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6494         break;
6495       case LOAD:
6496       case LOADLR:
6497         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6498         break;
6499       case STORE:
6500       case STORELR:
6501         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rs2[i],rs1[i],imm[i]);
6502         break;
6503       case ALU:
6504       case SHIFT:
6505         printf (" %x: %s r%d,r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i],rs2[i]);
6506         break;
6507       case MULTDIV:
6508         printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rs1[i],rs2[i]);
6509         break;
6510       case SHIFTIMM:
6511         printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6512         break;
6513       case MOV:
6514         if((opcode2[i]&0x1d)==0x10)
6515           printf (" %x: %s r%d\n",start+i*4,insn[i],rt1[i]);
6516         else if((opcode2[i]&0x1d)==0x11)
6517           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6518         else
6519           printf (" %x: %s\n",start+i*4,insn[i]);
6520         break;
6521       case COP0:
6522         if(opcode2[i]==0)
6523           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC0
6524         else if(opcode2[i]==4)
6525           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC0
6526         else printf (" %x: %s\n",start+i*4,insn[i]);
6527         break;
6528       case COP1:
6529         if(opcode2[i]<3)
6530           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC1
6531         else if(opcode2[i]>3)
6532           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC1
6533         else printf (" %x: %s\n",start+i*4,insn[i]);
6534         break;
6535       case COP2:
6536         if(opcode2[i]<3)
6537           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC2
6538         else if(opcode2[i]>3)
6539           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC2
6540         else printf (" %x: %s\n",start+i*4,insn[i]);
6541         break;
6542       case C1LS:
6543         printf (" %x: %s cpr1[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6544         break;
6545       case C2LS:
6546         printf (" %x: %s cpr2[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6547         break;
6548       case INTCALL:
6549         printf (" %x: %s (INTCALL)\n",start+i*4,insn[i]);
6550         break;
6551       default:
6552         //printf (" %s %8x\n",insn[i],source[i]);
6553         printf (" %x: %s\n",start+i*4,insn[i]);
6554     }
6555 }
6556 #else
6557 static void disassemble_inst(int i) {}
6558 #endif // DISASM
6559
6560 #define DRC_TEST_VAL 0x74657374
6561
6562 static void new_dynarec_test(void)
6563 {
6564   int (*testfunc)(void);
6565   void *beginning;
6566   int ret[2];
6567   size_t i;
6568
6569   // check structure linkage
6570   if ((u_char *)rcnts - (u_char *)&psxRegs != sizeof(psxRegs))
6571   {
6572     SysPrintf("linkage_arm* miscompilation/breakage detected.\n");
6573   }
6574
6575   SysPrintf("testing if we can run recompiled code...\n");
6576   ((volatile u_int *)out)[0]++; // make cache dirty
6577
6578   for (i = 0; i < ARRAY_SIZE(ret); i++) {
6579     out = ndrc->translation_cache;
6580     beginning = start_block();
6581     emit_movimm(DRC_TEST_VAL + i, 0); // test
6582     emit_ret();
6583     literal_pool(0);
6584     end_block(beginning);
6585     testfunc = beginning;
6586     ret[i] = testfunc();
6587   }
6588
6589   if (ret[0] == DRC_TEST_VAL && ret[1] == DRC_TEST_VAL + 1)
6590     SysPrintf("test passed.\n");
6591   else
6592     SysPrintf("test failed, will likely crash soon (r=%08x %08x)\n", ret[0], ret[1]);
6593   out = ndrc->translation_cache;
6594 }
6595
6596 // clear the state completely, instead of just marking
6597 // things invalid like invalidate_all_pages() does
6598 void new_dynarec_clear_full(void)
6599 {
6600   int n;
6601   out = ndrc->translation_cache;
6602   memset(invalid_code,1,sizeof(invalid_code));
6603   memset(hash_table,0xff,sizeof(hash_table));
6604   memset(mini_ht,-1,sizeof(mini_ht));
6605   memset(restore_candidate,0,sizeof(restore_candidate));
6606   memset(shadow,0,sizeof(shadow));
6607   copy=shadow;
6608   expirep=16384; // Expiry pointer, +2 blocks
6609   pending_exception=0;
6610   literalcount=0;
6611   stop_after_jal=0;
6612   inv_code_start=inv_code_end=~0;
6613   // TLB
6614   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6615   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6616   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6617 }
6618
6619 void new_dynarec_init(void)
6620 {
6621   SysPrintf("Init new dynarec\n");
6622
6623 #ifdef BASE_ADDR_DYNAMIC
6624   #ifdef VITA
6625   sceBlock = sceKernelAllocMemBlockForVM("code", 1 << TARGET_SIZE_2);
6626   if (sceBlock < 0)
6627     SysPrintf("sceKernelAllocMemBlockForVM failed\n");
6628   int ret = sceKernelGetMemBlockBase(sceBlock, (void **)&ndrc);
6629   if (ret < 0)
6630     SysPrintf("sceKernelGetMemBlockBase failed\n");
6631   #else
6632   uintptr_t desired_addr = 0;
6633   #ifdef __ELF__
6634   extern char _end;
6635   desired_addr = ((uintptr_t)&_end + 0xffffff) & ~0xffffffl;
6636   #endif
6637   ndrc = mmap((void *)desired_addr, sizeof(*ndrc),
6638             PROT_READ | PROT_WRITE | PROT_EXEC,
6639             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
6640   if (ndrc == MAP_FAILED) {
6641     SysPrintf("mmap() failed: %s\n", strerror(errno));
6642     abort();
6643   }
6644   #endif
6645 #else
6646   #ifndef NO_WRITE_EXEC
6647   // not all systems allow execute in data segment by default
6648   if (mprotect(ndrc, sizeof(ndrc->translation_cache) + sizeof(ndrc->tramp.ops),
6649                PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
6650     SysPrintf("mprotect() failed: %s\n", strerror(errno));
6651   #endif
6652 #endif
6653   out = ndrc->translation_cache;
6654   cycle_multiplier=200;
6655   new_dynarec_clear_full();
6656 #ifdef HOST_IMM8
6657   // Copy this into local area so we don't have to put it in every literal pool
6658   invc_ptr=invalid_code;
6659 #endif
6660   arch_init();
6661   new_dynarec_test();
6662 #ifndef RAM_FIXED
6663   ram_offset=(uintptr_t)rdram-0x80000000;
6664 #endif
6665   if (ram_offset!=0)
6666     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
6667 }
6668
6669 void new_dynarec_cleanup(void)
6670 {
6671   int n;
6672 #ifdef BASE_ADDR_DYNAMIC
6673   #ifdef VITA
6674   sceKernelFreeMemBlock(sceBlock);
6675   sceBlock = -1;
6676   #else
6677   if (munmap(ndrc, sizeof(*ndrc)) < 0)
6678     SysPrintf("munmap() failed\n");
6679   #endif
6680 #endif
6681   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6682   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6683   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6684   #ifdef ROM_COPY
6685   if (munmap (ROM_COPY, 67108864) < 0) {SysPrintf("munmap() failed\n");}
6686   #endif
6687 }
6688
6689 static u_int *get_source_start(u_int addr, u_int *limit)
6690 {
6691   if (!HACK_ENABLED(NDHACK_OVERRIDE_CYCLE_M))
6692     cycle_multiplier_override = 0;
6693
6694   if (addr < 0x00200000 ||
6695     (0xa0000000 <= addr && addr < 0xa0200000))
6696   {
6697     // used for BIOS calls mostly?
6698     *limit = (addr&0xa0000000)|0x00200000;
6699     return (u_int *)(rdram + (addr&0x1fffff));
6700   }
6701   else if (!Config.HLE && (
6702     /* (0x9fc00000 <= addr && addr < 0x9fc80000) ||*/
6703     (0xbfc00000 <= addr && addr < 0xbfc80000)))
6704   {
6705     // BIOS. The multiplier should be much higher as it's uncached 8bit mem,
6706     // but timings in PCSX are too tied to the interpreter's BIAS
6707     if (!HACK_ENABLED(NDHACK_OVERRIDE_CYCLE_M))
6708       cycle_multiplier_override = 200;
6709
6710     *limit = (addr & 0xfff00000) | 0x80000;
6711     return (u_int *)((u_char *)psxR + (addr&0x7ffff));
6712   }
6713   else if (addr >= 0x80000000 && addr < 0x80000000+RAM_SIZE) {
6714     *limit = (addr & 0x80600000) + 0x00200000;
6715     return (u_int *)(rdram + (addr&0x1fffff));
6716   }
6717   return NULL;
6718 }
6719
6720 static u_int scan_for_ret(u_int addr)
6721 {
6722   u_int limit = 0;
6723   u_int *mem;
6724
6725   mem = get_source_start(addr, &limit);
6726   if (mem == NULL)
6727     return addr;
6728
6729   if (limit > addr + 0x1000)
6730     limit = addr + 0x1000;
6731   for (; addr < limit; addr += 4, mem++) {
6732     if (*mem == 0x03e00008) // jr $ra
6733       return addr + 8;
6734   }
6735   return addr;
6736 }
6737
6738 struct savestate_block {
6739   uint32_t addr;
6740   uint32_t regflags;
6741 };
6742
6743 static int addr_cmp(const void *p1_, const void *p2_)
6744 {
6745   const struct savestate_block *p1 = p1_, *p2 = p2_;
6746   return p1->addr - p2->addr;
6747 }
6748
6749 int new_dynarec_save_blocks(void *save, int size)
6750 {
6751   struct savestate_block *blocks = save;
6752   int maxcount = size / sizeof(blocks[0]);
6753   struct savestate_block tmp_blocks[1024];
6754   struct ll_entry *head;
6755   int p, s, d, o, bcnt;
6756   u_int addr;
6757
6758   o = 0;
6759   for (p = 0; p < ARRAY_SIZE(jump_in); p++) {
6760     bcnt = 0;
6761     for (head = jump_in[p]; head != NULL; head = head->next) {
6762       tmp_blocks[bcnt].addr = head->vaddr;
6763       tmp_blocks[bcnt].regflags = head->reg_sv_flags;
6764       bcnt++;
6765     }
6766     if (bcnt < 1)
6767       continue;
6768     qsort(tmp_blocks, bcnt, sizeof(tmp_blocks[0]), addr_cmp);
6769
6770     addr = tmp_blocks[0].addr;
6771     for (s = d = 0; s < bcnt; s++) {
6772       if (tmp_blocks[s].addr < addr)
6773         continue;
6774       if (d == 0 || tmp_blocks[d-1].addr != tmp_blocks[s].addr)
6775         tmp_blocks[d++] = tmp_blocks[s];
6776       addr = scan_for_ret(tmp_blocks[s].addr);
6777     }
6778
6779     if (o + d > maxcount)
6780       d = maxcount - o;
6781     memcpy(&blocks[o], tmp_blocks, d * sizeof(blocks[0]));
6782     o += d;
6783   }
6784
6785   return o * sizeof(blocks[0]);
6786 }
6787
6788 void new_dynarec_load_blocks(const void *save, int size)
6789 {
6790   const struct savestate_block *blocks = save;
6791   int count = size / sizeof(blocks[0]);
6792   u_int regs_save[32];
6793   uint32_t f;
6794   int i, b;
6795
6796   get_addr(psxRegs.pc);
6797
6798   // change GPRs for speculation to at least partially work..
6799   memcpy(regs_save, &psxRegs.GPR, sizeof(regs_save));
6800   for (i = 1; i < 32; i++)
6801     psxRegs.GPR.r[i] = 0x80000000;
6802
6803   for (b = 0; b < count; b++) {
6804     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6805       if (f & 1)
6806         psxRegs.GPR.r[i] = 0x1f800000;
6807     }
6808
6809     get_addr(blocks[b].addr);
6810
6811     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6812       if (f & 1)
6813         psxRegs.GPR.r[i] = 0x80000000;
6814     }
6815   }
6816
6817   memcpy(&psxRegs.GPR, regs_save, sizeof(regs_save));
6818 }
6819
6820 int new_recompile_block(u_int addr)
6821 {
6822   u_int pagelimit = 0;
6823   u_int state_rflags = 0;
6824   int i;
6825
6826   assem_debug("NOTCOMPILED: addr = %x -> %p\n", addr, out);
6827   //printf("TRACE: count=%d next=%d (compile %x)\n",Count,next_interupt,addr);
6828   //if(debug)
6829   //printf("fpu mapping=%x enabled=%x\n",(Status & 0x04000000)>>26,(Status & 0x20000000)>>29);
6830
6831   // this is just for speculation
6832   for (i = 1; i < 32; i++) {
6833     if ((psxRegs.GPR.r[i] & 0xffff0000) == 0x1f800000)
6834       state_rflags |= 1 << i;
6835   }
6836
6837   start = (u_int)addr&~3;
6838   //assert(((u_int)addr&1)==0); // start-in-delay-slot flag
6839   new_dynarec_did_compile=1;
6840   if (Config.HLE && start == 0x80001000) // hlecall
6841   {
6842     // XXX: is this enough? Maybe check hleSoftCall?
6843     void *beginning=start_block();
6844     u_int page=get_page(start);
6845
6846     invalid_code[start>>12]=0;
6847     emit_movimm(start,0);
6848     emit_writeword(0,&pcaddr);
6849     emit_far_jump(new_dyna_leave);
6850     literal_pool(0);
6851     end_block(beginning);
6852     ll_add_flags(jump_in+page,start,state_rflags,(void *)beginning);
6853     return 0;
6854   }
6855
6856   source = get_source_start(start, &pagelimit);
6857   if (source == NULL) {
6858     SysPrintf("Compile at bogus memory address: %08x\n", addr);
6859     abort();
6860   }
6861
6862   /* Pass 1: disassemble */
6863   /* Pass 2: register dependencies, branch targets */
6864   /* Pass 3: register allocation */
6865   /* Pass 4: branch dependencies */
6866   /* Pass 5: pre-alloc */
6867   /* Pass 6: optimize clean/dirty state */
6868   /* Pass 7: flag 32-bit registers */
6869   /* Pass 8: assembly */
6870   /* Pass 9: linker */
6871   /* Pass 10: garbage collection / free memory */
6872
6873   int j;
6874   int done=0;
6875   unsigned int type,op,op2;
6876
6877   //printf("addr = %x source = %x %x\n", addr,source,source[0]);
6878
6879   /* Pass 1 disassembly */
6880
6881   for(i=0;!done;i++) {
6882     bt[i]=0;likely[i]=0;ooo[i]=0;op2=0;
6883     minimum_free_regs[i]=0;
6884     opcode[i]=op=source[i]>>26;
6885     switch(op)
6886     {
6887       case 0x00: strcpy(insn[i],"special"); type=NI;
6888         op2=source[i]&0x3f;
6889         switch(op2)
6890         {
6891           case 0x00: strcpy(insn[i],"SLL"); type=SHIFTIMM; break;
6892           case 0x02: strcpy(insn[i],"SRL"); type=SHIFTIMM; break;
6893           case 0x03: strcpy(insn[i],"SRA"); type=SHIFTIMM; break;
6894           case 0x04: strcpy(insn[i],"SLLV"); type=SHIFT; break;
6895           case 0x06: strcpy(insn[i],"SRLV"); type=SHIFT; break;
6896           case 0x07: strcpy(insn[i],"SRAV"); type=SHIFT; break;
6897           case 0x08: strcpy(insn[i],"JR"); type=RJUMP; break;
6898           case 0x09: strcpy(insn[i],"JALR"); type=RJUMP; break;
6899           case 0x0C: strcpy(insn[i],"SYSCALL"); type=SYSCALL; break;
6900           case 0x0D: strcpy(insn[i],"BREAK"); type=OTHER; break;
6901           case 0x0F: strcpy(insn[i],"SYNC"); type=OTHER; break;
6902           case 0x10: strcpy(insn[i],"MFHI"); type=MOV; break;
6903           case 0x11: strcpy(insn[i],"MTHI"); type=MOV; break;
6904           case 0x12: strcpy(insn[i],"MFLO"); type=MOV; break;
6905           case 0x13: strcpy(insn[i],"MTLO"); type=MOV; break;
6906           case 0x18: strcpy(insn[i],"MULT"); type=MULTDIV; break;
6907           case 0x19: strcpy(insn[i],"MULTU"); type=MULTDIV; break;
6908           case 0x1A: strcpy(insn[i],"DIV"); type=MULTDIV; break;
6909           case 0x1B: strcpy(insn[i],"DIVU"); type=MULTDIV; break;
6910           case 0x20: strcpy(insn[i],"ADD"); type=ALU; break;
6911           case 0x21: strcpy(insn[i],"ADDU"); type=ALU; break;
6912           case 0x22: strcpy(insn[i],"SUB"); type=ALU; break;
6913           case 0x23: strcpy(insn[i],"SUBU"); type=ALU; break;
6914           case 0x24: strcpy(insn[i],"AND"); type=ALU; break;
6915           case 0x25: strcpy(insn[i],"OR"); type=ALU; break;
6916           case 0x26: strcpy(insn[i],"XOR"); type=ALU; break;
6917           case 0x27: strcpy(insn[i],"NOR"); type=ALU; break;
6918           case 0x2A: strcpy(insn[i],"SLT"); type=ALU; break;
6919           case 0x2B: strcpy(insn[i],"SLTU"); type=ALU; break;
6920           case 0x30: strcpy(insn[i],"TGE"); type=NI; break;
6921           case 0x31: strcpy(insn[i],"TGEU"); type=NI; break;
6922           case 0x32: strcpy(insn[i],"TLT"); type=NI; break;
6923           case 0x33: strcpy(insn[i],"TLTU"); type=NI; break;
6924           case 0x34: strcpy(insn[i],"TEQ"); type=NI; break;
6925           case 0x36: strcpy(insn[i],"TNE"); type=NI; break;
6926 #if 0
6927           case 0x14: strcpy(insn[i],"DSLLV"); type=SHIFT; break;
6928           case 0x16: strcpy(insn[i],"DSRLV"); type=SHIFT; break;
6929           case 0x17: strcpy(insn[i],"DSRAV"); type=SHIFT; break;
6930           case 0x1C: strcpy(insn[i],"DMULT"); type=MULTDIV; break;
6931           case 0x1D: strcpy(insn[i],"DMULTU"); type=MULTDIV; break;
6932           case 0x1E: strcpy(insn[i],"DDIV"); type=MULTDIV; break;
6933           case 0x1F: strcpy(insn[i],"DDIVU"); type=MULTDIV; break;
6934           case 0x2C: strcpy(insn[i],"DADD"); type=ALU; break;
6935           case 0x2D: strcpy(insn[i],"DADDU"); type=ALU; break;
6936           case 0x2E: strcpy(insn[i],"DSUB"); type=ALU; break;
6937           case 0x2F: strcpy(insn[i],"DSUBU"); type=ALU; break;
6938           case 0x38: strcpy(insn[i],"DSLL"); type=SHIFTIMM; break;
6939           case 0x3A: strcpy(insn[i],"DSRL"); type=SHIFTIMM; break;
6940           case 0x3B: strcpy(insn[i],"DSRA"); type=SHIFTIMM; break;
6941           case 0x3C: strcpy(insn[i],"DSLL32"); type=SHIFTIMM; break;
6942           case 0x3E: strcpy(insn[i],"DSRL32"); type=SHIFTIMM; break;
6943           case 0x3F: strcpy(insn[i],"DSRA32"); type=SHIFTIMM; break;
6944 #endif
6945         }
6946         break;
6947       case 0x01: strcpy(insn[i],"regimm"); type=NI;
6948         op2=(source[i]>>16)&0x1f;
6949         switch(op2)
6950         {
6951           case 0x00: strcpy(insn[i],"BLTZ"); type=SJUMP; break;
6952           case 0x01: strcpy(insn[i],"BGEZ"); type=SJUMP; break;
6953           case 0x02: strcpy(insn[i],"BLTZL"); type=SJUMP; break;
6954           case 0x03: strcpy(insn[i],"BGEZL"); type=SJUMP; break;
6955           case 0x08: strcpy(insn[i],"TGEI"); type=NI; break;
6956           case 0x09: strcpy(insn[i],"TGEIU"); type=NI; break;
6957           case 0x0A: strcpy(insn[i],"TLTI"); type=NI; break;
6958           case 0x0B: strcpy(insn[i],"TLTIU"); type=NI; break;
6959           case 0x0C: strcpy(insn[i],"TEQI"); type=NI; break;
6960           case 0x0E: strcpy(insn[i],"TNEI"); type=NI; break;
6961           case 0x10: strcpy(insn[i],"BLTZAL"); type=SJUMP; break;
6962           case 0x11: strcpy(insn[i],"BGEZAL"); type=SJUMP; break;
6963           case 0x12: strcpy(insn[i],"BLTZALL"); type=SJUMP; break;
6964           case 0x13: strcpy(insn[i],"BGEZALL"); type=SJUMP; break;
6965         }
6966         break;
6967       case 0x02: strcpy(insn[i],"J"); type=UJUMP; break;
6968       case 0x03: strcpy(insn[i],"JAL"); type=UJUMP; break;
6969       case 0x04: strcpy(insn[i],"BEQ"); type=CJUMP; break;
6970       case 0x05: strcpy(insn[i],"BNE"); type=CJUMP; break;
6971       case 0x06: strcpy(insn[i],"BLEZ"); type=CJUMP; break;
6972       case 0x07: strcpy(insn[i],"BGTZ"); type=CJUMP; break;
6973       case 0x08: strcpy(insn[i],"ADDI"); type=IMM16; break;
6974       case 0x09: strcpy(insn[i],"ADDIU"); type=IMM16; break;
6975       case 0x0A: strcpy(insn[i],"SLTI"); type=IMM16; break;
6976       case 0x0B: strcpy(insn[i],"SLTIU"); type=IMM16; break;
6977       case 0x0C: strcpy(insn[i],"ANDI"); type=IMM16; break;
6978       case 0x0D: strcpy(insn[i],"ORI"); type=IMM16; break;
6979       case 0x0E: strcpy(insn[i],"XORI"); type=IMM16; break;
6980       case 0x0F: strcpy(insn[i],"LUI"); type=IMM16; break;
6981       case 0x10: strcpy(insn[i],"cop0"); type=NI;
6982         op2=(source[i]>>21)&0x1f;
6983         switch(op2)
6984         {
6985           case 0x00: strcpy(insn[i],"MFC0"); type=COP0; break;
6986           case 0x02: strcpy(insn[i],"CFC0"); type=COP0; break;
6987           case 0x04: strcpy(insn[i],"MTC0"); type=COP0; break;
6988           case 0x06: strcpy(insn[i],"CTC0"); type=COP0; break;
6989           case 0x10: strcpy(insn[i],"RFE"); type=COP0; break;
6990         }
6991         break;
6992       case 0x11: strcpy(insn[i],"cop1"); type=COP1;
6993         op2=(source[i]>>21)&0x1f;
6994         break;
6995 #if 0
6996       case 0x14: strcpy(insn[i],"BEQL"); type=CJUMP; break;
6997       case 0x15: strcpy(insn[i],"BNEL"); type=CJUMP; break;
6998       case 0x16: strcpy(insn[i],"BLEZL"); type=CJUMP; break;
6999       case 0x17: strcpy(insn[i],"BGTZL"); type=CJUMP; break;
7000       case 0x18: strcpy(insn[i],"DADDI"); type=IMM16; break;
7001       case 0x19: strcpy(insn[i],"DADDIU"); type=IMM16; break;
7002       case 0x1A: strcpy(insn[i],"LDL"); type=LOADLR; break;
7003       case 0x1B: strcpy(insn[i],"LDR"); type=LOADLR; break;
7004 #endif
7005       case 0x20: strcpy(insn[i],"LB"); type=LOAD; break;
7006       case 0x21: strcpy(insn[i],"LH"); type=LOAD; break;
7007       case 0x22: strcpy(insn[i],"LWL"); type=LOADLR; break;
7008       case 0x23: strcpy(insn[i],"LW"); type=LOAD; break;
7009       case 0x24: strcpy(insn[i],"LBU"); type=LOAD; break;
7010       case 0x25: strcpy(insn[i],"LHU"); type=LOAD; break;
7011       case 0x26: strcpy(insn[i],"LWR"); type=LOADLR; break;
7012 #if 0
7013       case 0x27: strcpy(insn[i],"LWU"); type=LOAD; break;
7014 #endif
7015       case 0x28: strcpy(insn[i],"SB"); type=STORE; break;
7016       case 0x29: strcpy(insn[i],"SH"); type=STORE; break;
7017       case 0x2A: strcpy(insn[i],"SWL"); type=STORELR; break;
7018       case 0x2B: strcpy(insn[i],"SW"); type=STORE; break;
7019 #if 0
7020       case 0x2C: strcpy(insn[i],"SDL"); type=STORELR; break;
7021       case 0x2D: strcpy(insn[i],"SDR"); type=STORELR; break;
7022 #endif
7023       case 0x2E: strcpy(insn[i],"SWR"); type=STORELR; break;
7024       case 0x2F: strcpy(insn[i],"CACHE"); type=NOP; break;
7025       case 0x30: strcpy(insn[i],"LL"); type=NI; break;
7026       case 0x31: strcpy(insn[i],"LWC1"); type=C1LS; break;
7027 #if 0
7028       case 0x34: strcpy(insn[i],"LLD"); type=NI; break;
7029       case 0x35: strcpy(insn[i],"LDC1"); type=C1LS; break;
7030       case 0x37: strcpy(insn[i],"LD"); type=LOAD; break;
7031 #endif
7032       case 0x38: strcpy(insn[i],"SC"); type=NI; break;
7033       case 0x39: strcpy(insn[i],"SWC1"); type=C1LS; break;
7034 #if 0
7035       case 0x3C: strcpy(insn[i],"SCD"); type=NI; break;
7036       case 0x3D: strcpy(insn[i],"SDC1"); type=C1LS; break;
7037       case 0x3F: strcpy(insn[i],"SD"); type=STORE; break;
7038 #endif
7039       case 0x12: strcpy(insn[i],"COP2"); type=NI;
7040         op2=(source[i]>>21)&0x1f;
7041         //if (op2 & 0x10)
7042         if (source[i]&0x3f) { // use this hack to support old savestates with patched gte insns
7043           if (gte_handlers[source[i]&0x3f]!=NULL) {
7044             if (gte_regnames[source[i]&0x3f]!=NULL)
7045               strcpy(insn[i],gte_regnames[source[i]&0x3f]);
7046             else
7047               snprintf(insn[i], sizeof(insn[i]), "COP2 %x", source[i]&0x3f);
7048             type=C2OP;
7049           }
7050         }
7051         else switch(op2)
7052         {
7053           case 0x00: strcpy(insn[i],"MFC2"); type=COP2; break;
7054           case 0x02: strcpy(insn[i],"CFC2"); type=COP2; break;
7055           case 0x04: strcpy(insn[i],"MTC2"); type=COP2; break;
7056           case 0x06: strcpy(insn[i],"CTC2"); type=COP2; break;
7057         }
7058         break;
7059       case 0x32: strcpy(insn[i],"LWC2"); type=C2LS; break;
7060       case 0x3A: strcpy(insn[i],"SWC2"); type=C2LS; break;
7061       case 0x3B: strcpy(insn[i],"HLECALL"); type=HLECALL; break;
7062       default: strcpy(insn[i],"???"); type=NI;
7063         SysPrintf("NI %08x @%08x (%08x)\n", source[i], addr + i*4, addr);
7064         break;
7065     }
7066     itype[i]=type;
7067     opcode2[i]=op2;
7068     /* Get registers/immediates */
7069     lt1[i]=0;
7070     dep1[i]=0;
7071     dep2[i]=0;
7072     gte_rs[i]=gte_rt[i]=0;
7073     switch(type) {
7074       case LOAD:
7075         rs1[i]=(source[i]>>21)&0x1f;
7076         rs2[i]=0;
7077         rt1[i]=(source[i]>>16)&0x1f;
7078         rt2[i]=0;
7079         imm[i]=(short)source[i];
7080         break;
7081       case STORE:
7082       case STORELR:
7083         rs1[i]=(source[i]>>21)&0x1f;
7084         rs2[i]=(source[i]>>16)&0x1f;
7085         rt1[i]=0;
7086         rt2[i]=0;
7087         imm[i]=(short)source[i];
7088         break;
7089       case LOADLR:
7090         // LWL/LWR only load part of the register,
7091         // therefore the target register must be treated as a source too
7092         rs1[i]=(source[i]>>21)&0x1f;
7093         rs2[i]=(source[i]>>16)&0x1f;
7094         rt1[i]=(source[i]>>16)&0x1f;
7095         rt2[i]=0;
7096         imm[i]=(short)source[i];
7097         if(op==0x26) dep1[i]=rt1[i]; // LWR
7098         break;
7099       case IMM16:
7100         if (op==0x0f) rs1[i]=0; // LUI instruction has no source register
7101         else rs1[i]=(source[i]>>21)&0x1f;
7102         rs2[i]=0;
7103         rt1[i]=(source[i]>>16)&0x1f;
7104         rt2[i]=0;
7105         if(op>=0x0c&&op<=0x0e) { // ANDI/ORI/XORI
7106           imm[i]=(unsigned short)source[i];
7107         }else{
7108           imm[i]=(short)source[i];
7109         }
7110         if(op==0x0d||op==0x0e) dep1[i]=rs1[i]; // ORI/XORI
7111         break;
7112       case UJUMP:
7113         rs1[i]=0;
7114         rs2[i]=0;
7115         rt1[i]=0;
7116         rt2[i]=0;
7117         // The JAL instruction writes to r31.
7118         if (op&1) {
7119           rt1[i]=31;
7120         }
7121         rs2[i]=CCREG;
7122         break;
7123       case RJUMP:
7124         rs1[i]=(source[i]>>21)&0x1f;
7125         rs2[i]=0;
7126         rt1[i]=0;
7127         rt2[i]=0;
7128         // The JALR instruction writes to rd.
7129         if (op2&1) {
7130           rt1[i]=(source[i]>>11)&0x1f;
7131         }
7132         rs2[i]=CCREG;
7133         break;
7134       case CJUMP:
7135         rs1[i]=(source[i]>>21)&0x1f;
7136         rs2[i]=(source[i]>>16)&0x1f;
7137         rt1[i]=0;
7138         rt2[i]=0;
7139         if(op&2) { // BGTZ/BLEZ
7140           rs2[i]=0;
7141         }
7142         likely[i]=op>>4;
7143         break;
7144       case SJUMP:
7145         rs1[i]=(source[i]>>21)&0x1f;
7146         rs2[i]=CCREG;
7147         rt1[i]=0;
7148         rt2[i]=0;
7149         if(op2&0x10) { // BxxAL
7150           rt1[i]=31;
7151           // NOTE: If the branch is not taken, r31 is still overwritten
7152         }
7153         likely[i]=(op2&2)>>1;
7154         break;
7155       case ALU:
7156         rs1[i]=(source[i]>>21)&0x1f; // source
7157         rs2[i]=(source[i]>>16)&0x1f; // subtract amount
7158         rt1[i]=(source[i]>>11)&0x1f; // destination
7159         rt2[i]=0;
7160         if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
7161           dep1[i]=rs1[i];dep2[i]=rs2[i];
7162         }
7163         else if(op2>=0x2c&&op2<=0x2f) { // DADD/DSUB
7164           dep1[i]=rs1[i];dep2[i]=rs2[i];
7165         }
7166         break;
7167       case MULTDIV:
7168         rs1[i]=(source[i]>>21)&0x1f; // source
7169         rs2[i]=(source[i]>>16)&0x1f; // divisor
7170         rt1[i]=HIREG;
7171         rt2[i]=LOREG;
7172         break;
7173       case MOV:
7174         rs1[i]=0;
7175         rs2[i]=0;
7176         rt1[i]=0;
7177         rt2[i]=0;
7178         if(op2==0x10) rs1[i]=HIREG; // MFHI
7179         if(op2==0x11) rt1[i]=HIREG; // MTHI
7180         if(op2==0x12) rs1[i]=LOREG; // MFLO
7181         if(op2==0x13) rt1[i]=LOREG; // MTLO
7182         if((op2&0x1d)==0x10) rt1[i]=(source[i]>>11)&0x1f; // MFxx
7183         if((op2&0x1d)==0x11) rs1[i]=(source[i]>>21)&0x1f; // MTxx
7184         dep1[i]=rs1[i];
7185         break;
7186       case SHIFT:
7187         rs1[i]=(source[i]>>16)&0x1f; // target of shift
7188         rs2[i]=(source[i]>>21)&0x1f; // shift amount
7189         rt1[i]=(source[i]>>11)&0x1f; // destination
7190         rt2[i]=0;
7191         break;
7192       case SHIFTIMM:
7193         rs1[i]=(source[i]>>16)&0x1f;
7194         rs2[i]=0;
7195         rt1[i]=(source[i]>>11)&0x1f;
7196         rt2[i]=0;
7197         imm[i]=(source[i]>>6)&0x1f;
7198         // DSxx32 instructions
7199         if(op2>=0x3c) imm[i]|=0x20;
7200         break;
7201       case COP0:
7202         rs1[i]=0;
7203         rs2[i]=0;
7204         rt1[i]=0;
7205         rt2[i]=0;
7206         if(op2==0||op2==2) rt1[i]=(source[i]>>16)&0x1F; // MFC0/CFC0
7207         if(op2==4||op2==6) rs1[i]=(source[i]>>16)&0x1F; // MTC0/CTC0
7208         if(op2==4&&((source[i]>>11)&0x1f)==12) rt2[i]=CSREG; // Status
7209         if(op2==16) if((source[i]&0x3f)==0x18) rs2[i]=CCREG; // ERET
7210         break;
7211       case COP1:
7212         rs1[i]=0;
7213         rs2[i]=0;
7214         rt1[i]=0;
7215         rt2[i]=0;
7216         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC1/DMFC1/CFC1
7217         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC1/DMTC1/CTC1
7218         rs2[i]=CSREG;
7219         break;
7220       case COP2:
7221         rs1[i]=0;
7222         rs2[i]=0;
7223         rt1[i]=0;
7224         rt2[i]=0;
7225         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC2/CFC2
7226         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC2/CTC2
7227         rs2[i]=CSREG;
7228         int gr=(source[i]>>11)&0x1F;
7229         switch(op2)
7230         {
7231           case 0x00: gte_rs[i]=1ll<<gr; break; // MFC2
7232           case 0x04: gte_rt[i]=1ll<<gr; break; // MTC2
7233           case 0x02: gte_rs[i]=1ll<<(gr+32); break; // CFC2
7234           case 0x06: gte_rt[i]=1ll<<(gr+32); break; // CTC2
7235         }
7236         break;
7237       case C1LS:
7238         rs1[i]=(source[i]>>21)&0x1F;
7239         rs2[i]=CSREG;
7240         rt1[i]=0;
7241         rt2[i]=0;
7242         imm[i]=(short)source[i];
7243         break;
7244       case C2LS:
7245         rs1[i]=(source[i]>>21)&0x1F;
7246         rs2[i]=0;
7247         rt1[i]=0;
7248         rt2[i]=0;
7249         imm[i]=(short)source[i];
7250         if(op==0x32) gte_rt[i]=1ll<<((source[i]>>16)&0x1F); // LWC2
7251         else gte_rs[i]=1ll<<((source[i]>>16)&0x1F); // SWC2
7252         break;
7253       case C2OP:
7254         rs1[i]=0;
7255         rs2[i]=0;
7256         rt1[i]=0;
7257         rt2[i]=0;
7258         gte_rs[i]=gte_reg_reads[source[i]&0x3f];
7259         gte_rt[i]=gte_reg_writes[source[i]&0x3f];
7260         gte_rt[i]|=1ll<<63; // every op changes flags
7261         if((source[i]&0x3f)==GTE_MVMVA) {
7262           int v = (source[i] >> 15) & 3;
7263           gte_rs[i]&=~0xe3fll;
7264           if(v==3) gte_rs[i]|=0xe00ll;
7265           else gte_rs[i]|=3ll<<(v*2);
7266         }
7267         break;
7268       case SYSCALL:
7269       case HLECALL:
7270       case INTCALL:
7271         rs1[i]=CCREG;
7272         rs2[i]=0;
7273         rt1[i]=0;
7274         rt2[i]=0;
7275         break;
7276       default:
7277         rs1[i]=0;
7278         rs2[i]=0;
7279         rt1[i]=0;
7280         rt2[i]=0;
7281     }
7282     /* Calculate branch target addresses */
7283     if(type==UJUMP)
7284       ba[i]=((start+i*4+4)&0xF0000000)|(((unsigned int)source[i]<<6)>>4);
7285     else if(type==CJUMP&&rs1[i]==rs2[i]&&(op&1))
7286       ba[i]=start+i*4+8; // Ignore never taken branch
7287     else if(type==SJUMP&&rs1[i]==0&&!(op2&1))
7288       ba[i]=start+i*4+8; // Ignore never taken branch
7289     else if(type==CJUMP||type==SJUMP)
7290       ba[i]=start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14);
7291     else ba[i]=-1;
7292     if (i > 0 && is_jump(i-1)) {
7293       int do_in_intrp=0;
7294       // branch in delay slot?
7295       if(type==RJUMP||type==UJUMP||type==CJUMP||type==SJUMP) {
7296         // don't handle first branch and call interpreter if it's hit
7297         SysPrintf("branch in delay slot @%08x (%08x)\n", addr + i*4, addr);
7298         do_in_intrp=1;
7299       }
7300       // basic load delay detection
7301       else if((type==LOAD||type==LOADLR||type==COP0||type==COP2||type==C2LS)&&rt1[i]!=0) {
7302         int t=(ba[i-1]-start)/4;
7303         if(0 <= t && t < i &&(rt1[i]==rs1[t]||rt1[i]==rs2[t])&&itype[t]!=CJUMP&&itype[t]!=SJUMP) {
7304           // jump target wants DS result - potential load delay effect
7305           SysPrintf("load delay @%08x (%08x)\n", addr + i*4, addr);
7306           do_in_intrp=1;
7307           bt[t+1]=1; // expected return from interpreter
7308         }
7309         else if(i>=2&&rt1[i-2]==2&&rt1[i]==2&&rs1[i]!=2&&rs2[i]!=2&&rs1[i-1]!=2&&rs2[i-1]!=2&&
7310               !(i>=3&&is_jump(i-3))) {
7311           // v0 overwrite like this is a sign of trouble, bail out
7312           SysPrintf("v0 overwrite @%08x (%08x)\n", addr + i*4, addr);
7313           do_in_intrp=1;
7314         }
7315       }
7316       if(do_in_intrp) {
7317         rs1[i-1]=CCREG;
7318         rs2[i-1]=rt1[i-1]=rt2[i-1]=0;
7319         ba[i-1]=-1;
7320         itype[i-1]=INTCALL;
7321         done=2;
7322         i--; // don't compile the DS
7323       }
7324     }
7325     /* Is this the end of the block? */
7326     if (i > 0 && is_ujump(i-1)) {
7327       if(rt1[i-1]==0) { // Continue past subroutine call (JAL)
7328         done=2;
7329       }
7330       else {
7331         if(stop_after_jal) done=1;
7332         // Stop on BREAK
7333         if((source[i+1]&0xfc00003f)==0x0d) done=1;
7334       }
7335       // Don't recompile stuff that's already compiled
7336       if(check_addr(start+i*4+4)) done=1;
7337       // Don't get too close to the limit
7338       if(i>MAXBLOCK/2) done=1;
7339     }
7340     if(itype[i]==SYSCALL&&stop_after_jal) done=1;
7341     if(itype[i]==HLECALL||itype[i]==INTCALL) done=2;
7342     if(done==2) {
7343       // Does the block continue due to a branch?
7344       for(j=i-1;j>=0;j--)
7345       {
7346         if(ba[j]==start+i*4) done=j=0; // Branch into delay slot
7347         if(ba[j]==start+i*4+4) done=j=0;
7348         if(ba[j]==start+i*4+8) done=j=0;
7349       }
7350     }
7351     //assert(i<MAXBLOCK-1);
7352     if(start+i*4==pagelimit-4) done=1;
7353     assert(start+i*4<pagelimit);
7354     if (i==MAXBLOCK-1) done=1;
7355     // Stop if we're compiling junk
7356     if(itype[i]==NI&&opcode[i]==0x11) {
7357       done=stop_after_jal=1;
7358       SysPrintf("Disabled speculative precompilation\n");
7359     }
7360   }
7361   slen=i;
7362   if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==RJUMP) {
7363     if(start+i*4==pagelimit) {
7364       itype[i-1]=SPAN;
7365     }
7366   }
7367   assert(slen>0);
7368
7369   /* Pass 2 - Register dependencies and branch targets */
7370
7371   unneeded_registers(0,slen-1,0);
7372
7373   /* Pass 3 - Register allocation */
7374
7375   struct regstat current; // Current register allocations/status
7376   current.dirty=0;
7377   current.u=unneeded_reg[0];
7378   clear_all_regs(current.regmap);
7379   alloc_reg(&current,0,CCREG);
7380   dirty_reg(&current,CCREG);
7381   current.isconst=0;
7382   current.wasconst=0;
7383   current.waswritten=0;
7384   int ds=0;
7385   int cc=0;
7386   int hr=-1;
7387
7388   if((u_int)addr&1) {
7389     // First instruction is delay slot
7390     cc=-1;
7391     bt[1]=1;
7392     ds=1;
7393     unneeded_reg[0]=1;
7394     current.regmap[HOST_BTREG]=BTREG;
7395   }
7396
7397   for(i=0;i<slen;i++)
7398   {
7399     if(bt[i])
7400     {
7401       int hr;
7402       for(hr=0;hr<HOST_REGS;hr++)
7403       {
7404         // Is this really necessary?
7405         if(current.regmap[hr]==0) current.regmap[hr]=-1;
7406       }
7407       current.isconst=0;
7408       current.waswritten=0;
7409     }
7410
7411     memcpy(regmap_pre[i],current.regmap,sizeof(current.regmap));
7412     regs[i].wasconst=current.isconst;
7413     regs[i].wasdirty=current.dirty;
7414     regs[i].loadedconst=0;
7415     if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP) {
7416       if(i+1<slen) {
7417         current.u=unneeded_reg[i+1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7418         current.u|=1;
7419       } else {
7420         current.u=1;
7421       }
7422     } else {
7423       if(i+1<slen) {
7424         current.u=branch_unneeded_reg[i]&~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
7425         current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7426         current.u|=1;
7427       } else { SysPrintf("oops, branch at end of block with no delay slot\n");abort(); }
7428     }
7429     is_ds[i]=ds;
7430     if(ds) {
7431       ds=0; // Skip delay slot, already allocated as part of branch
7432       // ...but we need to alloc it in case something jumps here
7433       if(i+1<slen) {
7434         current.u=branch_unneeded_reg[i-1]&unneeded_reg[i+1];
7435       }else{
7436         current.u=branch_unneeded_reg[i-1];
7437       }
7438       current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7439       current.u|=1;
7440       struct regstat temp;
7441       memcpy(&temp,&current,sizeof(current));
7442       temp.wasdirty=temp.dirty;
7443       // TODO: Take into account unconditional branches, as below
7444       delayslot_alloc(&temp,i);
7445       memcpy(regs[i].regmap,temp.regmap,sizeof(temp.regmap));
7446       regs[i].wasdirty=temp.wasdirty;
7447       regs[i].dirty=temp.dirty;
7448       regs[i].isconst=0;
7449       regs[i].wasconst=0;
7450       current.isconst=0;
7451       // Create entry (branch target) regmap
7452       for(hr=0;hr<HOST_REGS;hr++)
7453       {
7454         int r=temp.regmap[hr];
7455         if(r>=0) {
7456           if(r!=regmap_pre[i][hr]) {
7457             regs[i].regmap_entry[hr]=-1;
7458           }
7459           else
7460           {
7461               assert(r < 64);
7462               if((current.u>>r)&1) {
7463                 regs[i].regmap_entry[hr]=-1;
7464                 regs[i].regmap[hr]=-1;
7465                 //Don't clear regs in the delay slot as the branch might need them
7466                 //current.regmap[hr]=-1;
7467               }else
7468                 regs[i].regmap_entry[hr]=r;
7469           }
7470         } else {
7471           // First instruction expects CCREG to be allocated
7472           if(i==0&&hr==HOST_CCREG)
7473             regs[i].regmap_entry[hr]=CCREG;
7474           else
7475             regs[i].regmap_entry[hr]=-1;
7476         }
7477       }
7478     }
7479     else { // Not delay slot
7480       switch(itype[i]) {
7481         case UJUMP:
7482           //current.isconst=0; // DEBUG
7483           //current.wasconst=0; // DEBUG
7484           //regs[i].wasconst=0; // DEBUG
7485           clear_const(&current,rt1[i]);
7486           alloc_cc(&current,i);
7487           dirty_reg(&current,CCREG);
7488           if (rt1[i]==31) {
7489             alloc_reg(&current,i,31);
7490             dirty_reg(&current,31);
7491             //assert(rs1[i+1]!=31&&rs2[i+1]!=31);
7492             //assert(rt1[i+1]!=rt1[i]);
7493             #ifdef REG_PREFETCH
7494             alloc_reg(&current,i,PTEMP);
7495             #endif
7496           }
7497           ooo[i]=1;
7498           delayslot_alloc(&current,i+1);
7499           //current.isconst=0; // DEBUG
7500           ds=1;
7501           //printf("i=%d, isconst=%x\n",i,current.isconst);
7502           break;
7503         case RJUMP:
7504           //current.isconst=0;
7505           //current.wasconst=0;
7506           //regs[i].wasconst=0;
7507           clear_const(&current,rs1[i]);
7508           clear_const(&current,rt1[i]);
7509           alloc_cc(&current,i);
7510           dirty_reg(&current,CCREG);
7511           if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
7512             alloc_reg(&current,i,rs1[i]);
7513             if (rt1[i]!=0) {
7514               alloc_reg(&current,i,rt1[i]);
7515               dirty_reg(&current,rt1[i]);
7516               assert(rs1[i+1]!=rt1[i]&&rs2[i+1]!=rt1[i]);
7517               assert(rt1[i+1]!=rt1[i]);
7518               #ifdef REG_PREFETCH
7519               alloc_reg(&current,i,PTEMP);
7520               #endif
7521             }
7522             #ifdef USE_MINI_HT
7523             if(rs1[i]==31) { // JALR
7524               alloc_reg(&current,i,RHASH);
7525               alloc_reg(&current,i,RHTBL);
7526             }
7527             #endif
7528             delayslot_alloc(&current,i+1);
7529           } else {
7530             // The delay slot overwrites our source register,
7531             // allocate a temporary register to hold the old value.
7532             current.isconst=0;
7533             current.wasconst=0;
7534             regs[i].wasconst=0;
7535             delayslot_alloc(&current,i+1);
7536             current.isconst=0;
7537             alloc_reg(&current,i,RTEMP);
7538           }
7539           //current.isconst=0; // DEBUG
7540           ooo[i]=1;
7541           ds=1;
7542           break;
7543         case CJUMP:
7544           //current.isconst=0;
7545           //current.wasconst=0;
7546           //regs[i].wasconst=0;
7547           clear_const(&current,rs1[i]);
7548           clear_const(&current,rs2[i]);
7549           if((opcode[i]&0x3E)==4) // BEQ/BNE
7550           {
7551             alloc_cc(&current,i);
7552             dirty_reg(&current,CCREG);
7553             if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7554             if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7555             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]))||
7556                (rs2[i]&&(rs2[i]==rt1[i+1]||rs2[i]==rt2[i+1]))) {
7557               // The delay slot overwrites one of our conditions.
7558               // Allocate the branch condition registers instead.
7559               current.isconst=0;
7560               current.wasconst=0;
7561               regs[i].wasconst=0;
7562               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7563               if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7564             }
7565             else
7566             {
7567               ooo[i]=1;
7568               delayslot_alloc(&current,i+1);
7569             }
7570           }
7571           else
7572           if((opcode[i]&0x3E)==6) // BLEZ/BGTZ
7573           {
7574             alloc_cc(&current,i);
7575             dirty_reg(&current,CCREG);
7576             alloc_reg(&current,i,rs1[i]);
7577             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
7578               // The delay slot overwrites one of our conditions.
7579               // Allocate the branch condition registers instead.
7580               current.isconst=0;
7581               current.wasconst=0;
7582               regs[i].wasconst=0;
7583               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7584             }
7585             else
7586             {
7587               ooo[i]=1;
7588               delayslot_alloc(&current,i+1);
7589             }
7590           }
7591           else
7592           // Don't alloc the delay slot yet because we might not execute it
7593           if((opcode[i]&0x3E)==0x14) // BEQL/BNEL
7594           {
7595             current.isconst=0;
7596             current.wasconst=0;
7597             regs[i].wasconst=0;
7598             alloc_cc(&current,i);
7599             dirty_reg(&current,CCREG);
7600             alloc_reg(&current,i,rs1[i]);
7601             alloc_reg(&current,i,rs2[i]);
7602           }
7603           else
7604           if((opcode[i]&0x3E)==0x16) // BLEZL/BGTZL
7605           {
7606             current.isconst=0;
7607             current.wasconst=0;
7608             regs[i].wasconst=0;
7609             alloc_cc(&current,i);
7610             dirty_reg(&current,CCREG);
7611             alloc_reg(&current,i,rs1[i]);
7612           }
7613           ds=1;
7614           //current.isconst=0;
7615           break;
7616         case SJUMP:
7617           //current.isconst=0;
7618           //current.wasconst=0;
7619           //regs[i].wasconst=0;
7620           clear_const(&current,rs1[i]);
7621           clear_const(&current,rt1[i]);
7622           //if((opcode2[i]&0x1E)==0x0) // BLTZ/BGEZ
7623           if((opcode2[i]&0x0E)==0x0) // BLTZ/BGEZ
7624           {
7625             alloc_cc(&current,i);
7626             dirty_reg(&current,CCREG);
7627             alloc_reg(&current,i,rs1[i]);
7628             if (rt1[i]==31) { // BLTZAL/BGEZAL
7629               alloc_reg(&current,i,31);
7630               dirty_reg(&current,31);
7631               //#ifdef REG_PREFETCH
7632               //alloc_reg(&current,i,PTEMP);
7633               //#endif
7634             }
7635             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) // The delay slot overwrites the branch condition.
7636                ||(rt1[i]==31&&(rs1[i+1]==31||rs2[i+1]==31||rt1[i+1]==31||rt2[i+1]==31))) { // DS touches $ra
7637               // Allocate the branch condition registers instead.
7638               current.isconst=0;
7639               current.wasconst=0;
7640               regs[i].wasconst=0;
7641               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7642             }
7643             else
7644             {
7645               ooo[i]=1;
7646               delayslot_alloc(&current,i+1);
7647             }
7648           }
7649           else
7650           // Don't alloc the delay slot yet because we might not execute it
7651           if((opcode2[i]&0x1E)==0x2) // BLTZL/BGEZL
7652           {
7653             current.isconst=0;
7654             current.wasconst=0;
7655             regs[i].wasconst=0;
7656             alloc_cc(&current,i);
7657             dirty_reg(&current,CCREG);
7658             alloc_reg(&current,i,rs1[i]);
7659           }
7660           ds=1;
7661           //current.isconst=0;
7662           break;
7663         case IMM16:
7664           imm16_alloc(&current,i);
7665           break;
7666         case LOAD:
7667         case LOADLR:
7668           load_alloc(&current,i);
7669           break;
7670         case STORE:
7671         case STORELR:
7672           store_alloc(&current,i);
7673           break;
7674         case ALU:
7675           alu_alloc(&current,i);
7676           break;
7677         case SHIFT:
7678           shift_alloc(&current,i);
7679           break;
7680         case MULTDIV:
7681           multdiv_alloc(&current,i);
7682           break;
7683         case SHIFTIMM:
7684           shiftimm_alloc(&current,i);
7685           break;
7686         case MOV:
7687           mov_alloc(&current,i);
7688           break;
7689         case COP0:
7690           cop0_alloc(&current,i);
7691           break;
7692         case COP1:
7693           break;
7694         case COP2:
7695           cop2_alloc(&current,i);
7696           break;
7697         case C1LS:
7698           c1ls_alloc(&current,i);
7699           break;
7700         case C2LS:
7701           c2ls_alloc(&current,i);
7702           break;
7703         case C2OP:
7704           c2op_alloc(&current,i);
7705           break;
7706         case SYSCALL:
7707         case HLECALL:
7708         case INTCALL:
7709           syscall_alloc(&current,i);
7710           break;
7711         case SPAN:
7712           pagespan_alloc(&current,i);
7713           break;
7714       }
7715
7716       // Create entry (branch target) regmap
7717       for(hr=0;hr<HOST_REGS;hr++)
7718       {
7719         int r,or;
7720         r=current.regmap[hr];
7721         if(r>=0) {
7722           if(r!=regmap_pre[i][hr]) {
7723             // TODO: delay slot (?)
7724             or=get_reg(regmap_pre[i],r); // Get old mapping for this register
7725             if(or<0||(r&63)>=TEMPREG){
7726               regs[i].regmap_entry[hr]=-1;
7727             }
7728             else
7729             {
7730               // Just move it to a different register
7731               regs[i].regmap_entry[hr]=r;
7732               // If it was dirty before, it's still dirty
7733               if((regs[i].wasdirty>>or)&1) dirty_reg(&current,r&63);
7734             }
7735           }
7736           else
7737           {
7738             // Unneeded
7739             if(r==0){
7740               regs[i].regmap_entry[hr]=0;
7741             }
7742             else
7743             {
7744               assert(r<64);
7745               if((current.u>>r)&1) {
7746                 regs[i].regmap_entry[hr]=-1;
7747                 //regs[i].regmap[hr]=-1;
7748                 current.regmap[hr]=-1;
7749               }else
7750                 regs[i].regmap_entry[hr]=r;
7751             }
7752           }
7753         } else {
7754           // Branches expect CCREG to be allocated at the target
7755           if(regmap_pre[i][hr]==CCREG)
7756             regs[i].regmap_entry[hr]=CCREG;
7757           else
7758             regs[i].regmap_entry[hr]=-1;
7759         }
7760       }
7761       memcpy(regs[i].regmap,current.regmap,sizeof(current.regmap));
7762     }
7763
7764     if(i>0&&(itype[i-1]==STORE||itype[i-1]==STORELR||(itype[i-1]==C2LS&&opcode[i-1]==0x3a))&&(u_int)imm[i-1]<0x800)
7765       current.waswritten|=1<<rs1[i-1];
7766     current.waswritten&=~(1<<rt1[i]);
7767     current.waswritten&=~(1<<rt2[i]);
7768     if((itype[i]==STORE||itype[i]==STORELR||(itype[i]==C2LS&&opcode[i]==0x3a))&&(u_int)imm[i]>=0x800)
7769       current.waswritten&=~(1<<rs1[i]);
7770
7771     /* Branch post-alloc */
7772     if(i>0)
7773     {
7774       current.wasdirty=current.dirty;
7775       switch(itype[i-1]) {
7776         case UJUMP:
7777           memcpy(&branch_regs[i-1],&current,sizeof(current));
7778           branch_regs[i-1].isconst=0;
7779           branch_regs[i-1].wasconst=0;
7780           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7781           alloc_cc(&branch_regs[i-1],i-1);
7782           dirty_reg(&branch_regs[i-1],CCREG);
7783           if(rt1[i-1]==31) { // JAL
7784             alloc_reg(&branch_regs[i-1],i-1,31);
7785             dirty_reg(&branch_regs[i-1],31);
7786           }
7787           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7788           memcpy(constmap[i],constmap[i-1],sizeof(constmap[i]));
7789           break;
7790         case RJUMP:
7791           memcpy(&branch_regs[i-1],&current,sizeof(current));
7792           branch_regs[i-1].isconst=0;
7793           branch_regs[i-1].wasconst=0;
7794           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7795           alloc_cc(&branch_regs[i-1],i-1);
7796           dirty_reg(&branch_regs[i-1],CCREG);
7797           alloc_reg(&branch_regs[i-1],i-1,rs1[i-1]);
7798           if(rt1[i-1]!=0) { // JALR
7799             alloc_reg(&branch_regs[i-1],i-1,rt1[i-1]);
7800             dirty_reg(&branch_regs[i-1],rt1[i-1]);
7801           }
7802           #ifdef USE_MINI_HT
7803           if(rs1[i-1]==31) { // JALR
7804             alloc_reg(&branch_regs[i-1],i-1,RHASH);
7805             alloc_reg(&branch_regs[i-1],i-1,RHTBL);
7806           }
7807           #endif
7808           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7809           memcpy(constmap[i],constmap[i-1],sizeof(constmap[i]));
7810           break;
7811         case CJUMP:
7812           if((opcode[i-1]&0x3E)==4) // BEQ/BNE
7813           {
7814             alloc_cc(&current,i-1);
7815             dirty_reg(&current,CCREG);
7816             if((rs1[i-1]&&(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]))||
7817                (rs2[i-1]&&(rs2[i-1]==rt1[i]||rs2[i-1]==rt2[i]))) {
7818               // The delay slot overwrote one of our conditions
7819               // Delay slot goes after the test (in order)
7820               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7821               current.u|=1;
7822               delayslot_alloc(&current,i);
7823               current.isconst=0;
7824             }
7825             else
7826             {
7827               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7828               // Alloc the branch condition registers
7829               if(rs1[i-1]) alloc_reg(&current,i-1,rs1[i-1]);
7830               if(rs2[i-1]) alloc_reg(&current,i-1,rs2[i-1]);
7831             }
7832             memcpy(&branch_regs[i-1],&current,sizeof(current));
7833             branch_regs[i-1].isconst=0;
7834             branch_regs[i-1].wasconst=0;
7835             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7836             memcpy(constmap[i],constmap[i-1],sizeof(constmap[i]));
7837           }
7838           else
7839           if((opcode[i-1]&0x3E)==6) // BLEZ/BGTZ
7840           {
7841             alloc_cc(&current,i-1);
7842             dirty_reg(&current,CCREG);
7843             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7844               // The delay slot overwrote the branch condition
7845               // Delay slot goes after the test (in order)
7846               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7847               current.u|=1;
7848               delayslot_alloc(&current,i);
7849               current.isconst=0;
7850             }
7851             else
7852             {
7853               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7854               // Alloc the branch condition register
7855               alloc_reg(&current,i-1,rs1[i-1]);
7856             }
7857             memcpy(&branch_regs[i-1],&current,sizeof(current));
7858             branch_regs[i-1].isconst=0;
7859             branch_regs[i-1].wasconst=0;
7860             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7861             memcpy(constmap[i],constmap[i-1],sizeof(constmap[i]));
7862           }
7863           else
7864           // Alloc the delay slot in case the branch is taken
7865           if((opcode[i-1]&0x3E)==0x14) // BEQL/BNEL
7866           {
7867             memcpy(&branch_regs[i-1],&current,sizeof(current));
7868             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7869             alloc_cc(&branch_regs[i-1],i);
7870             dirty_reg(&branch_regs[i-1],CCREG);
7871             delayslot_alloc(&branch_regs[i-1],i);
7872             branch_regs[i-1].isconst=0;
7873             alloc_reg(&current,i,CCREG); // Not taken path
7874             dirty_reg(&current,CCREG);
7875             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7876           }
7877           else
7878           if((opcode[i-1]&0x3E)==0x16) // BLEZL/BGTZL
7879           {
7880             memcpy(&branch_regs[i-1],&current,sizeof(current));
7881             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7882             alloc_cc(&branch_regs[i-1],i);
7883             dirty_reg(&branch_regs[i-1],CCREG);
7884             delayslot_alloc(&branch_regs[i-1],i);
7885             branch_regs[i-1].isconst=0;
7886             alloc_reg(&current,i,CCREG); // Not taken path
7887             dirty_reg(&current,CCREG);
7888             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7889           }
7890           break;
7891         case SJUMP:
7892           //if((opcode2[i-1]&0x1E)==0) // BLTZ/BGEZ
7893           if((opcode2[i-1]&0x0E)==0) // BLTZ/BGEZ
7894           {
7895             alloc_cc(&current,i-1);
7896             dirty_reg(&current,CCREG);
7897             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7898               // The delay slot overwrote the branch condition
7899               // Delay slot goes after the test (in order)
7900               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7901               current.u|=1;
7902               delayslot_alloc(&current,i);
7903               current.isconst=0;
7904             }
7905             else
7906             {
7907               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7908               // Alloc the branch condition register
7909               alloc_reg(&current,i-1,rs1[i-1]);
7910             }
7911             memcpy(&branch_regs[i-1],&current,sizeof(current));
7912             branch_regs[i-1].isconst=0;
7913             branch_regs[i-1].wasconst=0;
7914             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7915             memcpy(constmap[i],constmap[i-1],sizeof(constmap[i]));
7916           }
7917           else
7918           // Alloc the delay slot in case the branch is taken
7919           if((opcode2[i-1]&0x1E)==2) // BLTZL/BGEZL
7920           {
7921             memcpy(&branch_regs[i-1],&current,sizeof(current));
7922             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7923             alloc_cc(&branch_regs[i-1],i);
7924             dirty_reg(&branch_regs[i-1],CCREG);
7925             delayslot_alloc(&branch_regs[i-1],i);
7926             branch_regs[i-1].isconst=0;
7927             alloc_reg(&current,i,CCREG); // Not taken path
7928             dirty_reg(&current,CCREG);
7929             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7930           }
7931           // FIXME: BLTZAL/BGEZAL
7932           if(opcode2[i-1]&0x10) { // BxxZAL
7933             alloc_reg(&branch_regs[i-1],i-1,31);
7934             dirty_reg(&branch_regs[i-1],31);
7935           }
7936           break;
7937       }
7938
7939       if (is_ujump(i-1))
7940       {
7941         if(rt1[i-1]==31) // JAL/JALR
7942         {
7943           // Subroutine call will return here, don't alloc any registers
7944           current.dirty=0;
7945           clear_all_regs(current.regmap);
7946           alloc_reg(&current,i,CCREG);
7947           dirty_reg(&current,CCREG);
7948         }
7949         else if(i+1<slen)
7950         {
7951           // Internal branch will jump here, match registers to caller
7952           current.dirty=0;
7953           clear_all_regs(current.regmap);
7954           alloc_reg(&current,i,CCREG);
7955           dirty_reg(&current,CCREG);
7956           for(j=i-1;j>=0;j--)
7957           {
7958             if(ba[j]==start+i*4+4) {
7959               memcpy(current.regmap,branch_regs[j].regmap,sizeof(current.regmap));
7960               current.dirty=branch_regs[j].dirty;
7961               break;
7962             }
7963           }
7964           while(j>=0) {
7965             if(ba[j]==start+i*4+4) {
7966               for(hr=0;hr<HOST_REGS;hr++) {
7967                 if(current.regmap[hr]!=branch_regs[j].regmap[hr]) {
7968                   current.regmap[hr]=-1;
7969                 }
7970                 current.dirty&=branch_regs[j].dirty;
7971               }
7972             }
7973             j--;
7974           }
7975         }
7976       }
7977     }
7978
7979     // Count cycles in between branches
7980     ccadj[i]=cc;
7981     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i]==SYSCALL||itype[i]==HLECALL))
7982     {
7983       cc=0;
7984     }
7985 #if !defined(DRC_DBG)
7986     else if(itype[i]==C2OP&&gte_cycletab[source[i]&0x3f]>2)
7987     {
7988       // this should really be removed since the real stalls have been implemented,
7989       // but doing so causes sizeable perf regression against the older version
7990       u_int gtec = gte_cycletab[source[i] & 0x3f];
7991       cc += HACK_ENABLED(NDHACK_GTE_NO_STALL) ? gtec/2 : 2;
7992     }
7993     else if(i>1&&itype[i]==STORE&&itype[i-1]==STORE&&itype[i-2]==STORE&&!bt[i])
7994     {
7995       cc+=4;
7996     }
7997     else if(itype[i]==C2LS)
7998     {
7999       // same as with C2OP
8000       cc += HACK_ENABLED(NDHACK_GTE_NO_STALL) ? 4 : 2;
8001     }
8002 #endif
8003     else
8004     {
8005       cc++;
8006     }
8007
8008     if(!is_ds[i]) {
8009       regs[i].dirty=current.dirty;
8010       regs[i].isconst=current.isconst;
8011       memcpy(constmap[i],current_constmap,sizeof(constmap[i]));
8012     }
8013     for(hr=0;hr<HOST_REGS;hr++) {
8014       if(hr!=EXCLUDE_REG&&regs[i].regmap[hr]>=0) {
8015         if(regmap_pre[i][hr]!=regs[i].regmap[hr]) {
8016           regs[i].wasconst&=~(1<<hr);
8017         }
8018       }
8019     }
8020     if(current.regmap[HOST_BTREG]==BTREG) current.regmap[HOST_BTREG]=-1;
8021     regs[i].waswritten=current.waswritten;
8022   }
8023
8024   /* Pass 4 - Cull unused host registers */
8025
8026   uint64_t nr=0;
8027
8028   for (i=slen-1;i>=0;i--)
8029   {
8030     int hr;
8031     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8032     {
8033       if(ba[i]<start || ba[i]>=(start+slen*4))
8034       {
8035         // Branch out of this block, don't need anything
8036         nr=0;
8037       }
8038       else
8039       {
8040         // Internal branch
8041         // Need whatever matches the target
8042         nr=0;
8043         int t=(ba[i]-start)>>2;
8044         for(hr=0;hr<HOST_REGS;hr++)
8045         {
8046           if(regs[i].regmap_entry[hr]>=0) {
8047             if(regs[i].regmap_entry[hr]==regs[t].regmap_entry[hr]) nr|=1<<hr;
8048           }
8049         }
8050       }
8051       // Conditional branch may need registers for following instructions
8052       if (!is_ujump(i))
8053       {
8054         if(i<slen-2) {
8055           nr|=needed_reg[i+2];
8056           for(hr=0;hr<HOST_REGS;hr++)
8057           {
8058             if(regmap_pre[i+2][hr]>=0&&get_reg(regs[i+2].regmap_entry,regmap_pre[i+2][hr])<0) nr&=~(1<<hr);
8059             //if((regmap_entry[i+2][hr])>=0) if(!((nr>>hr)&1)) printf("%x-bogus(%d=%d)\n",start+i*4,hr,regmap_entry[i+2][hr]);
8060           }
8061         }
8062       }
8063       // Don't need stuff which is overwritten
8064       //if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
8065       //if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
8066       // Merge in delay slot
8067       for(hr=0;hr<HOST_REGS;hr++)
8068       {
8069         if(!likely[i]) {
8070           // These are overwritten unless the branch is "likely"
8071           // and the delay slot is nullified if not taken
8072           if(rt1[i+1]&&rt1[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8073           if(rt2[i+1]&&rt2[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8074         }
8075         if(rs1[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
8076         if(rs2[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
8077         if(rs1[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8078         if(rs2[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8079         if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) {
8080           if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
8081           if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
8082         }
8083       }
8084     }
8085     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
8086     {
8087       // SYSCALL instruction (software interrupt)
8088       nr=0;
8089     }
8090     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
8091     {
8092       // ERET instruction (return from interrupt)
8093       nr=0;
8094     }
8095     else // Non-branch
8096     {
8097       if(i<slen-1) {
8098         for(hr=0;hr<HOST_REGS;hr++) {
8099           if(regmap_pre[i+1][hr]>=0&&get_reg(regs[i+1].regmap_entry,regmap_pre[i+1][hr])<0) nr&=~(1<<hr);
8100           if(regs[i].regmap[hr]!=regmap_pre[i+1][hr]) nr&=~(1<<hr);
8101           if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
8102           if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
8103         }
8104       }
8105     }
8106     for(hr=0;hr<HOST_REGS;hr++)
8107     {
8108       // Overwritten registers are not needed
8109       if(rt1[i]&&rt1[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8110       if(rt2[i]&&rt2[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8111       if(FTEMP==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8112       // Source registers are needed
8113       if(rs1[i]==regmap_pre[i][hr]) nr|=1<<hr;
8114       if(rs2[i]==regmap_pre[i][hr]) nr|=1<<hr;
8115       if(rs1[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8116       if(rs2[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8117       if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) {
8118         if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
8119         if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
8120       }
8121       // Don't store a register immediately after writing it,
8122       // may prevent dual-issue.
8123       // But do so if this is a branch target, otherwise we
8124       // might have to load the register before the branch.
8125       if(i>0&&!bt[i]&&((regs[i].wasdirty>>hr)&1)) {
8126         if((regmap_pre[i][hr]>0&&!((unneeded_reg[i]>>regmap_pre[i][hr])&1))) {
8127           if(rt1[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
8128           if(rt2[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
8129         }
8130         if((regs[i].regmap_entry[hr]>0&&!((unneeded_reg[i]>>regs[i].regmap_entry[hr])&1))) {
8131           if(rt1[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
8132           if(rt2[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
8133         }
8134       }
8135     }
8136     // Cycle count is needed at branches.  Assume it is needed at the target too.
8137     if(i==0||bt[i]||itype[i]==CJUMP||itype[i]==SPAN) {
8138       if(regmap_pre[i][HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
8139       if(regs[i].regmap_entry[HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
8140     }
8141     // Save it
8142     needed_reg[i]=nr;
8143
8144     // Deallocate unneeded registers
8145     for(hr=0;hr<HOST_REGS;hr++)
8146     {
8147       if(!((nr>>hr)&1)) {
8148         if(regs[i].regmap_entry[hr]!=CCREG) regs[i].regmap_entry[hr]=-1;
8149         if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
8150            (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8151            (regs[i].regmap[hr]&63)!=PTEMP && (regs[i].regmap[hr]&63)!=CCREG)
8152         {
8153           if (!is_ujump(i))
8154           {
8155             if(likely[i]) {
8156               regs[i].regmap[hr]=-1;
8157               regs[i].isconst&=~(1<<hr);
8158               if(i<slen-2) {
8159                 regmap_pre[i+2][hr]=-1;
8160                 regs[i+2].wasconst&=~(1<<hr);
8161               }
8162             }
8163           }
8164         }
8165         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8166         {
8167           int map=0,temp=0;
8168           if(itype[i+1]==STORE || itype[i+1]==STORELR ||
8169              (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
8170             map=INVCP;
8171           }
8172           if(itype[i+1]==LOADLR || itype[i+1]==STORELR ||
8173              itype[i+1]==C1LS || itype[i+1]==C2LS)
8174             temp=FTEMP;
8175           if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
8176              (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8177              (regs[i].regmap[hr]&63)!=rt1[i+1] && (regs[i].regmap[hr]&63)!=rt2[i+1] &&
8178              regs[i].regmap[hr]!=rs1[i+1] && regs[i].regmap[hr]!=rs2[i+1] &&
8179              (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=PTEMP &&
8180              regs[i].regmap[hr]!=RHASH && regs[i].regmap[hr]!=RHTBL &&
8181              regs[i].regmap[hr]!=RTEMP && regs[i].regmap[hr]!=CCREG &&
8182              regs[i].regmap[hr]!=map )
8183           {
8184             regs[i].regmap[hr]=-1;
8185             regs[i].isconst&=~(1<<hr);
8186             if((branch_regs[i].regmap[hr]&63)!=rs1[i] && (branch_regs[i].regmap[hr]&63)!=rs2[i] &&
8187                (branch_regs[i].regmap[hr]&63)!=rt1[i] && (branch_regs[i].regmap[hr]&63)!=rt2[i] &&
8188                (branch_regs[i].regmap[hr]&63)!=rt1[i+1] && (branch_regs[i].regmap[hr]&63)!=rt2[i+1] &&
8189                branch_regs[i].regmap[hr]!=rs1[i+1] && branch_regs[i].regmap[hr]!=rs2[i+1] &&
8190                (branch_regs[i].regmap[hr]&63)!=temp && branch_regs[i].regmap[hr]!=PTEMP &&
8191                branch_regs[i].regmap[hr]!=RHASH && branch_regs[i].regmap[hr]!=RHTBL &&
8192                branch_regs[i].regmap[hr]!=RTEMP && branch_regs[i].regmap[hr]!=CCREG &&
8193                branch_regs[i].regmap[hr]!=map)
8194             {
8195               branch_regs[i].regmap[hr]=-1;
8196               branch_regs[i].regmap_entry[hr]=-1;
8197               if (!is_ujump(i))
8198               {
8199                 if(!likely[i]&&i<slen-2) {
8200                   regmap_pre[i+2][hr]=-1;
8201                   regs[i+2].wasconst&=~(1<<hr);
8202                 }
8203               }
8204             }
8205           }
8206         }
8207         else
8208         {
8209           // Non-branch
8210           if(i>0)
8211           {
8212             int map=-1,temp=-1;
8213             if(itype[i]==STORE || itype[i]==STORELR ||
8214                       (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
8215               map=INVCP;
8216             }
8217             if(itype[i]==LOADLR || itype[i]==STORELR ||
8218                itype[i]==C1LS || itype[i]==C2LS)
8219               temp=FTEMP;
8220             if((regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8221                regs[i].regmap[hr]!=rs1[i] && regs[i].regmap[hr]!=rs2[i] &&
8222                (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=map &&
8223                (itype[i]!=SPAN||regs[i].regmap[hr]!=CCREG))
8224             {
8225               if(i<slen-1&&!is_ds[i]) {
8226                 assert(regs[i].regmap[hr]<64);
8227                 if(regmap_pre[i+1][hr]!=-1 || regs[i].regmap[hr]>0)
8228                 if(regmap_pre[i+1][hr]!=regs[i].regmap[hr])
8229                 {
8230                   SysPrintf("fail: %x (%d %d!=%d)\n",start+i*4,hr,regmap_pre[i+1][hr],regs[i].regmap[hr]);
8231                   assert(regmap_pre[i+1][hr]==regs[i].regmap[hr]);
8232                 }
8233                 regmap_pre[i+1][hr]=-1;
8234                 if(regs[i+1].regmap_entry[hr]==CCREG) regs[i+1].regmap_entry[hr]=-1;
8235                 regs[i+1].wasconst&=~(1<<hr);
8236               }
8237               regs[i].regmap[hr]=-1;
8238               regs[i].isconst&=~(1<<hr);
8239             }
8240           }
8241         }
8242       } // if needed
8243     } // for hr
8244   }
8245
8246   /* Pass 5 - Pre-allocate registers */
8247
8248   // If a register is allocated during a loop, try to allocate it for the
8249   // entire loop, if possible.  This avoids loading/storing registers
8250   // inside of the loop.
8251
8252   signed char f_regmap[HOST_REGS];
8253   clear_all_regs(f_regmap);
8254   for(i=0;i<slen-1;i++)
8255   {
8256     if(itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8257     {
8258       if(ba[i]>=start && ba[i]<(start+i*4))
8259       if(itype[i+1]==NOP||itype[i+1]==MOV||itype[i+1]==ALU
8260       ||itype[i+1]==SHIFTIMM||itype[i+1]==IMM16||itype[i+1]==LOAD
8261       ||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS
8262       ||itype[i+1]==SHIFT||itype[i+1]==COP1
8263       ||itype[i+1]==COP2||itype[i+1]==C2LS||itype[i+1]==C2OP)
8264       {
8265         int t=(ba[i]-start)>>2;
8266         if(t>0&&(itype[t-1]!=UJUMP&&itype[t-1]!=RJUMP&&itype[t-1]!=CJUMP&&itype[t-1]!=SJUMP)) // loop_preload can't handle jumps into delay slots
8267         if(t<2||(itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||rt1[t-2]!=31) // call/ret assumes no registers allocated
8268         for(hr=0;hr<HOST_REGS;hr++)
8269         {
8270           if(regs[i].regmap[hr]>=0) {
8271             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8272               // dealloc old register
8273               int n;
8274               for(n=0;n<HOST_REGS;n++)
8275               {
8276                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8277               }
8278               // and alloc new one
8279               f_regmap[hr]=regs[i].regmap[hr];
8280             }
8281           }
8282           if(branch_regs[i].regmap[hr]>=0) {
8283             if(f_regmap[hr]!=branch_regs[i].regmap[hr]) {
8284               // dealloc old register
8285               int n;
8286               for(n=0;n<HOST_REGS;n++)
8287               {
8288                 if(f_regmap[n]==branch_regs[i].regmap[hr]) {f_regmap[n]=-1;}
8289               }
8290               // and alloc new one
8291               f_regmap[hr]=branch_regs[i].regmap[hr];
8292             }
8293           }
8294           if(ooo[i]) {
8295             if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1])
8296               f_regmap[hr]=branch_regs[i].regmap[hr];
8297           }else{
8298             if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1])
8299               f_regmap[hr]=branch_regs[i].regmap[hr];
8300           }
8301           // Avoid dirty->clean transition
8302           #ifdef DESTRUCTIVE_WRITEBACK
8303           if(t>0) if(get_reg(regmap_pre[t],f_regmap[hr])>=0) if((regs[t].wasdirty>>get_reg(regmap_pre[t],f_regmap[hr]))&1) f_regmap[hr]=-1;
8304           #endif
8305           // This check is only strictly required in the DESTRUCTIVE_WRITEBACK
8306           // case above, however it's always a good idea.  We can't hoist the
8307           // load if the register was already allocated, so there's no point
8308           // wasting time analyzing most of these cases.  It only "succeeds"
8309           // when the mapping was different and the load can be replaced with
8310           // a mov, which is of negligible benefit.  So such cases are
8311           // skipped below.
8312           if(f_regmap[hr]>0) {
8313             if(regs[t].regmap[hr]==f_regmap[hr]||(regs[t].regmap_entry[hr]<0&&get_reg(regmap_pre[t],f_regmap[hr])<0)) {
8314               int r=f_regmap[hr];
8315               for(j=t;j<=i;j++)
8316               {
8317                 //printf("Test %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8318                 if(r<34&&((unneeded_reg[j]>>r)&1)) break;
8319                 assert(r < 64);
8320                 if(regs[j].regmap[hr]==f_regmap[hr]&&(f_regmap[hr]&63)<TEMPREG) {
8321                   //printf("Hit %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8322                   int k;
8323                   if(regs[i].regmap[hr]==-1&&branch_regs[i].regmap[hr]==-1) {
8324                     if(get_reg(regs[i+2].regmap,f_regmap[hr])>=0) break;
8325                     if(r>63) {
8326                       if(get_reg(regs[i].regmap,r&63)<0) break;
8327                       if(get_reg(branch_regs[i].regmap,r&63)<0) break;
8328                     }
8329                     k=i;
8330                     while(k>1&&regs[k-1].regmap[hr]==-1) {
8331                       if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8332                         //printf("no free regs for store %x\n",start+(k-1)*4);
8333                         break;
8334                       }
8335                       if(get_reg(regs[k-1].regmap,f_regmap[hr])>=0) {
8336                         //printf("no-match due to different register\n");
8337                         break;
8338                       }
8339                       if(itype[k-2]==UJUMP||itype[k-2]==RJUMP||itype[k-2]==CJUMP||itype[k-2]==SJUMP) {
8340                         //printf("no-match due to branch\n");
8341                         break;
8342                       }
8343                       // call/ret fast path assumes no registers allocated
8344                       if(k>2&&(itype[k-3]==UJUMP||itype[k-3]==RJUMP)&&rt1[k-3]==31) {
8345                         break;
8346                       }
8347                       assert(r < 64);
8348                       k--;
8349                     }
8350                     if(regs[k-1].regmap[hr]==f_regmap[hr]&&regmap_pre[k][hr]==f_regmap[hr]) {
8351                       //printf("Extend r%d, %x ->\n",hr,start+k*4);
8352                       while(k<i) {
8353                         regs[k].regmap_entry[hr]=f_regmap[hr];
8354                         regs[k].regmap[hr]=f_regmap[hr];
8355                         regmap_pre[k+1][hr]=f_regmap[hr];
8356                         regs[k].wasdirty&=~(1<<hr);
8357                         regs[k].dirty&=~(1<<hr);
8358                         regs[k].wasdirty|=(1<<hr)&regs[k-1].dirty;
8359                         regs[k].dirty|=(1<<hr)&regs[k].wasdirty;
8360                         regs[k].wasconst&=~(1<<hr);
8361                         regs[k].isconst&=~(1<<hr);
8362                         k++;
8363                       }
8364                     }
8365                     else {
8366                       //printf("Fail Extend r%d, %x ->\n",hr,start+k*4);
8367                       break;
8368                     }
8369                     assert(regs[i-1].regmap[hr]==f_regmap[hr]);
8370                     if(regs[i-1].regmap[hr]==f_regmap[hr]&&regmap_pre[i][hr]==f_regmap[hr]) {
8371                       //printf("OK fill %x (r%d)\n",start+i*4,hr);
8372                       regs[i].regmap_entry[hr]=f_regmap[hr];
8373                       regs[i].regmap[hr]=f_regmap[hr];
8374                       regs[i].wasdirty&=~(1<<hr);
8375                       regs[i].dirty&=~(1<<hr);
8376                       regs[i].wasdirty|=(1<<hr)&regs[i-1].dirty;
8377                       regs[i].dirty|=(1<<hr)&regs[i-1].dirty;
8378                       regs[i].wasconst&=~(1<<hr);
8379                       regs[i].isconst&=~(1<<hr);
8380                       branch_regs[i].regmap_entry[hr]=f_regmap[hr];
8381                       branch_regs[i].wasdirty&=~(1<<hr);
8382                       branch_regs[i].wasdirty|=(1<<hr)&regs[i].dirty;
8383                       branch_regs[i].regmap[hr]=f_regmap[hr];
8384                       branch_regs[i].dirty&=~(1<<hr);
8385                       branch_regs[i].dirty|=(1<<hr)&regs[i].dirty;
8386                       branch_regs[i].wasconst&=~(1<<hr);
8387                       branch_regs[i].isconst&=~(1<<hr);
8388                       if (!is_ujump(i)) {
8389                         regmap_pre[i+2][hr]=f_regmap[hr];
8390                         regs[i+2].wasdirty&=~(1<<hr);
8391                         regs[i+2].wasdirty|=(1<<hr)&regs[i].dirty;
8392                       }
8393                     }
8394                   }
8395                   for(k=t;k<j;k++) {
8396                     // Alloc register clean at beginning of loop,
8397                     // but may dirty it in pass 6
8398                     regs[k].regmap_entry[hr]=f_regmap[hr];
8399                     regs[k].regmap[hr]=f_regmap[hr];
8400                     regs[k].dirty&=~(1<<hr);
8401                     regs[k].wasconst&=~(1<<hr);
8402                     regs[k].isconst&=~(1<<hr);
8403                     if(itype[k]==UJUMP||itype[k]==RJUMP||itype[k]==CJUMP||itype[k]==SJUMP) {
8404                       branch_regs[k].regmap_entry[hr]=f_regmap[hr];
8405                       branch_regs[k].regmap[hr]=f_regmap[hr];
8406                       branch_regs[k].dirty&=~(1<<hr);
8407                       branch_regs[k].wasconst&=~(1<<hr);
8408                       branch_regs[k].isconst&=~(1<<hr);
8409                       if (!is_ujump(k)) {
8410                         regmap_pre[k+2][hr]=f_regmap[hr];
8411                         regs[k+2].wasdirty&=~(1<<hr);
8412                       }
8413                     }
8414                     else
8415                     {
8416                       regmap_pre[k+1][hr]=f_regmap[hr];
8417                       regs[k+1].wasdirty&=~(1<<hr);
8418                     }
8419                   }
8420                   if(regs[j].regmap[hr]==f_regmap[hr])
8421                     regs[j].regmap_entry[hr]=f_regmap[hr];
8422                   break;
8423                 }
8424                 if(j==i) break;
8425                 if(regs[j].regmap[hr]>=0)
8426                   break;
8427                 if(get_reg(regs[j].regmap,f_regmap[hr])>=0) {
8428                   //printf("no-match due to different register\n");
8429                   break;
8430                 }
8431                 if (is_ujump(j))
8432                 {
8433                   // Stop on unconditional branch
8434                   break;
8435                 }
8436                 if(itype[j]==CJUMP||itype[j]==SJUMP)
8437                 {
8438                   if(ooo[j]) {
8439                     if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1])
8440                       break;
8441                   }else{
8442                     if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1])
8443                       break;
8444                   }
8445                   if(get_reg(branch_regs[j].regmap,f_regmap[hr])>=0) {
8446                     //printf("no-match due to different register (branch)\n");
8447                     break;
8448                   }
8449                 }
8450                 if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8451                   //printf("No free regs for store %x\n",start+j*4);
8452                   break;
8453                 }
8454                 assert(f_regmap[hr]<64);
8455               }
8456             }
8457           }
8458         }
8459       }
8460     }else{
8461       // Non branch or undetermined branch target
8462       for(hr=0;hr<HOST_REGS;hr++)
8463       {
8464         if(hr!=EXCLUDE_REG) {
8465           if(regs[i].regmap[hr]>=0) {
8466             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8467               // dealloc old register
8468               int n;
8469               for(n=0;n<HOST_REGS;n++)
8470               {
8471                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8472               }
8473               // and alloc new one
8474               f_regmap[hr]=regs[i].regmap[hr];
8475             }
8476           }
8477         }
8478       }
8479       // Try to restore cycle count at branch targets
8480       if(bt[i]) {
8481         for(j=i;j<slen-1;j++) {
8482           if(regs[j].regmap[HOST_CCREG]!=-1) break;
8483           if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8484             //printf("no free regs for store %x\n",start+j*4);
8485             break;
8486           }
8487         }
8488         if(regs[j].regmap[HOST_CCREG]==CCREG) {
8489           int k=i;
8490           //printf("Extend CC, %x -> %x\n",start+k*4,start+j*4);
8491           while(k<j) {
8492             regs[k].regmap_entry[HOST_CCREG]=CCREG;
8493             regs[k].regmap[HOST_CCREG]=CCREG;
8494             regmap_pre[k+1][HOST_CCREG]=CCREG;
8495             regs[k+1].wasdirty|=1<<HOST_CCREG;
8496             regs[k].dirty|=1<<HOST_CCREG;
8497             regs[k].wasconst&=~(1<<HOST_CCREG);
8498             regs[k].isconst&=~(1<<HOST_CCREG);
8499             k++;
8500           }
8501           regs[j].regmap_entry[HOST_CCREG]=CCREG;
8502         }
8503         // Work backwards from the branch target
8504         if(j>i&&f_regmap[HOST_CCREG]==CCREG)
8505         {
8506           //printf("Extend backwards\n");
8507           int k;
8508           k=i;
8509           while(regs[k-1].regmap[HOST_CCREG]==-1) {
8510             if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8511               //printf("no free regs for store %x\n",start+(k-1)*4);
8512               break;
8513             }
8514             k--;
8515           }
8516           if(regs[k-1].regmap[HOST_CCREG]==CCREG) {
8517             //printf("Extend CC, %x ->\n",start+k*4);
8518             while(k<=i) {
8519               regs[k].regmap_entry[HOST_CCREG]=CCREG;
8520               regs[k].regmap[HOST_CCREG]=CCREG;
8521               regmap_pre[k+1][HOST_CCREG]=CCREG;
8522               regs[k+1].wasdirty|=1<<HOST_CCREG;
8523               regs[k].dirty|=1<<HOST_CCREG;
8524               regs[k].wasconst&=~(1<<HOST_CCREG);
8525               regs[k].isconst&=~(1<<HOST_CCREG);
8526               k++;
8527             }
8528           }
8529           else {
8530             //printf("Fail Extend CC, %x ->\n",start+k*4);
8531           }
8532         }
8533       }
8534       if(itype[i]!=STORE&&itype[i]!=STORELR&&itype[i]!=C1LS&&itype[i]!=SHIFT&&
8535          itype[i]!=NOP&&itype[i]!=MOV&&itype[i]!=ALU&&itype[i]!=SHIFTIMM&&
8536          itype[i]!=IMM16&&itype[i]!=LOAD&&itype[i]!=COP1)
8537       {
8538         memcpy(f_regmap,regs[i].regmap,sizeof(f_regmap));
8539       }
8540     }
8541   }
8542
8543   // This allocates registers (if possible) one instruction prior
8544   // to use, which can avoid a load-use penalty on certain CPUs.
8545   for(i=0;i<slen-1;i++)
8546   {
8547     if(!i||(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP))
8548     {
8549       if(!bt[i+1])
8550       {
8551         if(itype[i]==ALU||itype[i]==MOV||itype[i]==LOAD||itype[i]==SHIFTIMM||itype[i]==IMM16
8552            ||((itype[i]==COP1||itype[i]==COP2)&&opcode2[i]<3))
8553         {
8554           if(rs1[i+1]) {
8555             if((hr=get_reg(regs[i+1].regmap,rs1[i+1]))>=0)
8556             {
8557               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8558               {
8559                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8560                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8561                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8562                 regs[i].isconst&=~(1<<hr);
8563                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8564                 constmap[i][hr]=constmap[i+1][hr];
8565                 regs[i+1].wasdirty&=~(1<<hr);
8566                 regs[i].dirty&=~(1<<hr);
8567               }
8568             }
8569           }
8570           if(rs2[i+1]) {
8571             if((hr=get_reg(regs[i+1].regmap,rs2[i+1]))>=0)
8572             {
8573               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8574               {
8575                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8576                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8577                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8578                 regs[i].isconst&=~(1<<hr);
8579                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8580                 constmap[i][hr]=constmap[i+1][hr];
8581                 regs[i+1].wasdirty&=~(1<<hr);
8582                 regs[i].dirty&=~(1<<hr);
8583               }
8584             }
8585           }
8586           // Preload target address for load instruction (non-constant)
8587           if(itype[i+1]==LOAD&&rs1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8588             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8589             {
8590               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8591               {
8592                 regs[i].regmap[hr]=rs1[i+1];
8593                 regmap_pre[i+1][hr]=rs1[i+1];
8594                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8595                 regs[i].isconst&=~(1<<hr);
8596                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8597                 constmap[i][hr]=constmap[i+1][hr];
8598                 regs[i+1].wasdirty&=~(1<<hr);
8599                 regs[i].dirty&=~(1<<hr);
8600               }
8601             }
8602           }
8603           // Load source into target register
8604           if(lt1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8605             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8606             {
8607               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8608               {
8609                 regs[i].regmap[hr]=rs1[i+1];
8610                 regmap_pre[i+1][hr]=rs1[i+1];
8611                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8612                 regs[i].isconst&=~(1<<hr);
8613                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8614                 constmap[i][hr]=constmap[i+1][hr];
8615                 regs[i+1].wasdirty&=~(1<<hr);
8616                 regs[i].dirty&=~(1<<hr);
8617               }
8618             }
8619           }
8620           // Address for store instruction (non-constant)
8621           if(itype[i+1]==STORE||itype[i+1]==STORELR
8622              ||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SB/SH/SW/SD/SWC1/SDC1/SWC2/SDC2
8623             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8624               hr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1);
8625               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8626               else {regs[i+1].regmap[hr]=AGEN1+((i+1)&1);regs[i+1].isconst&=~(1<<hr);}
8627               assert(hr>=0);
8628               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8629               {
8630                 regs[i].regmap[hr]=rs1[i+1];
8631                 regmap_pre[i+1][hr]=rs1[i+1];
8632                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8633                 regs[i].isconst&=~(1<<hr);
8634                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8635                 constmap[i][hr]=constmap[i+1][hr];
8636                 regs[i+1].wasdirty&=~(1<<hr);
8637                 regs[i].dirty&=~(1<<hr);
8638               }
8639             }
8640           }
8641           if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) { // LWC1/LDC1, LWC2/LDC2
8642             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8643               int nr;
8644               hr=get_reg(regs[i+1].regmap,FTEMP);
8645               assert(hr>=0);
8646               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8647               {
8648                 regs[i].regmap[hr]=rs1[i+1];
8649                 regmap_pre[i+1][hr]=rs1[i+1];
8650                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8651                 regs[i].isconst&=~(1<<hr);
8652                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8653                 constmap[i][hr]=constmap[i+1][hr];
8654                 regs[i+1].wasdirty&=~(1<<hr);
8655                 regs[i].dirty&=~(1<<hr);
8656               }
8657               else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
8658               {
8659                 // move it to another register
8660                 regs[i+1].regmap[hr]=-1;
8661                 regmap_pre[i+2][hr]=-1;
8662                 regs[i+1].regmap[nr]=FTEMP;
8663                 regmap_pre[i+2][nr]=FTEMP;
8664                 regs[i].regmap[nr]=rs1[i+1];
8665                 regmap_pre[i+1][nr]=rs1[i+1];
8666                 regs[i+1].regmap_entry[nr]=rs1[i+1];
8667                 regs[i].isconst&=~(1<<nr);
8668                 regs[i+1].isconst&=~(1<<nr);
8669                 regs[i].dirty&=~(1<<nr);
8670                 regs[i+1].wasdirty&=~(1<<nr);
8671                 regs[i+1].dirty&=~(1<<nr);
8672                 regs[i+2].wasdirty&=~(1<<nr);
8673               }
8674             }
8675           }
8676           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR/*||itype[i+1]==C1LS||||itype[i+1]==C2LS*/) {
8677             if(itype[i+1]==LOAD)
8678               hr=get_reg(regs[i+1].regmap,rt1[i+1]);
8679             if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) // LWC1/LDC1, LWC2/LDC2
8680               hr=get_reg(regs[i+1].regmap,FTEMP);
8681             if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1/SWC2/SDC2
8682               hr=get_reg(regs[i+1].regmap,AGEN1+((i+1)&1));
8683               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8684             }
8685             if(hr>=0&&regs[i].regmap[hr]<0) {
8686               int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
8687               if(rs>=0&&((regs[i+1].wasconst>>rs)&1)) {
8688                 regs[i].regmap[hr]=AGEN1+((i+1)&1);
8689                 regmap_pre[i+1][hr]=AGEN1+((i+1)&1);
8690                 regs[i+1].regmap_entry[hr]=AGEN1+((i+1)&1);
8691                 regs[i].isconst&=~(1<<hr);
8692                 regs[i+1].wasdirty&=~(1<<hr);
8693                 regs[i].dirty&=~(1<<hr);
8694               }
8695             }
8696           }
8697         }
8698       }
8699     }
8700   }
8701
8702   /* Pass 6 - Optimize clean/dirty state */
8703   clean_registers(0,slen-1,1);
8704
8705   /* Pass 7 - Identify 32-bit registers */
8706   for (i=slen-1;i>=0;i--)
8707   {
8708     if(itype[i]==CJUMP||itype[i]==SJUMP)
8709     {
8710       // Conditional branch
8711       if((source[i]>>16)!=0x1000&&i<slen-2) {
8712         // Mark this address as a branch target since it may be called
8713         // upon return from interrupt
8714         bt[i+2]=1;
8715       }
8716     }
8717   }
8718
8719   if(itype[slen-1]==SPAN) {
8720     bt[slen-1]=1; // Mark as a branch target so instruction can restart after exception
8721   }
8722
8723 #ifdef DISASM
8724   /* Debug/disassembly */
8725   for(i=0;i<slen;i++)
8726   {
8727     printf("U:");
8728     int r;
8729     for(r=1;r<=CCREG;r++) {
8730       if((unneeded_reg[i]>>r)&1) {
8731         if(r==HIREG) printf(" HI");
8732         else if(r==LOREG) printf(" LO");
8733         else printf(" r%d",r);
8734       }
8735     }
8736     printf("\n");
8737     #if defined(__i386__) || defined(__x86_64__)
8738     printf("pre: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7]);
8739     #endif
8740     #ifdef __arm__
8741     printf("pre: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][4],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7],regmap_pre[i][8],regmap_pre[i][9],regmap_pre[i][10],regmap_pre[i][12]);
8742     #endif
8743     #if defined(__i386__) || defined(__x86_64__)
8744     printf("needs: ");
8745     if(needed_reg[i]&1) printf("eax ");
8746     if((needed_reg[i]>>1)&1) printf("ecx ");
8747     if((needed_reg[i]>>2)&1) printf("edx ");
8748     if((needed_reg[i]>>3)&1) printf("ebx ");
8749     if((needed_reg[i]>>5)&1) printf("ebp ");
8750     if((needed_reg[i]>>6)&1) printf("esi ");
8751     if((needed_reg[i]>>7)&1) printf("edi ");
8752     printf("\n");
8753     printf("entry: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7]);
8754     printf("dirty: ");
8755     if(regs[i].wasdirty&1) printf("eax ");
8756     if((regs[i].wasdirty>>1)&1) printf("ecx ");
8757     if((regs[i].wasdirty>>2)&1) printf("edx ");
8758     if((regs[i].wasdirty>>3)&1) printf("ebx ");
8759     if((regs[i].wasdirty>>5)&1) printf("ebp ");
8760     if((regs[i].wasdirty>>6)&1) printf("esi ");
8761     if((regs[i].wasdirty>>7)&1) printf("edi ");
8762     #endif
8763     #ifdef __arm__
8764     printf("entry: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[4],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7],regs[i].regmap_entry[8],regs[i].regmap_entry[9],regs[i].regmap_entry[10],regs[i].regmap_entry[12]);
8765     printf("dirty: ");
8766     if(regs[i].wasdirty&1) printf("r0 ");
8767     if((regs[i].wasdirty>>1)&1) printf("r1 ");
8768     if((regs[i].wasdirty>>2)&1) printf("r2 ");
8769     if((regs[i].wasdirty>>3)&1) printf("r3 ");
8770     if((regs[i].wasdirty>>4)&1) printf("r4 ");
8771     if((regs[i].wasdirty>>5)&1) printf("r5 ");
8772     if((regs[i].wasdirty>>6)&1) printf("r6 ");
8773     if((regs[i].wasdirty>>7)&1) printf("r7 ");
8774     if((regs[i].wasdirty>>8)&1) printf("r8 ");
8775     if((regs[i].wasdirty>>9)&1) printf("r9 ");
8776     if((regs[i].wasdirty>>10)&1) printf("r10 ");
8777     if((regs[i].wasdirty>>12)&1) printf("r12 ");
8778     #endif
8779     printf("\n");
8780     disassemble_inst(i);
8781     //printf ("ccadj[%d] = %d\n",i,ccadj[i]);
8782     #if defined(__i386__) || defined(__x86_64__)
8783     printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7]);
8784     if(regs[i].dirty&1) printf("eax ");
8785     if((regs[i].dirty>>1)&1) printf("ecx ");
8786     if((regs[i].dirty>>2)&1) printf("edx ");
8787     if((regs[i].dirty>>3)&1) printf("ebx ");
8788     if((regs[i].dirty>>5)&1) printf("ebp ");
8789     if((regs[i].dirty>>6)&1) printf("esi ");
8790     if((regs[i].dirty>>7)&1) printf("edi ");
8791     #endif
8792     #ifdef __arm__
8793     printf("r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[4],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7],regs[i].regmap[8],regs[i].regmap[9],regs[i].regmap[10],regs[i].regmap[12]);
8794     if(regs[i].dirty&1) printf("r0 ");
8795     if((regs[i].dirty>>1)&1) printf("r1 ");
8796     if((regs[i].dirty>>2)&1) printf("r2 ");
8797     if((regs[i].dirty>>3)&1) printf("r3 ");
8798     if((regs[i].dirty>>4)&1) printf("r4 ");
8799     if((regs[i].dirty>>5)&1) printf("r5 ");
8800     if((regs[i].dirty>>6)&1) printf("r6 ");
8801     if((regs[i].dirty>>7)&1) printf("r7 ");
8802     if((regs[i].dirty>>8)&1) printf("r8 ");
8803     if((regs[i].dirty>>9)&1) printf("r9 ");
8804     if((regs[i].dirty>>10)&1) printf("r10 ");
8805     if((regs[i].dirty>>12)&1) printf("r12 ");
8806     #endif
8807     printf("\n");
8808     if(regs[i].isconst) {
8809       printf("constants: ");
8810       #if defined(__i386__) || defined(__x86_64__)
8811       if(regs[i].isconst&1) printf("eax=%x ",(u_int)constmap[i][0]);
8812       if((regs[i].isconst>>1)&1) printf("ecx=%x ",(u_int)constmap[i][1]);
8813       if((regs[i].isconst>>2)&1) printf("edx=%x ",(u_int)constmap[i][2]);
8814       if((regs[i].isconst>>3)&1) printf("ebx=%x ",(u_int)constmap[i][3]);
8815       if((regs[i].isconst>>5)&1) printf("ebp=%x ",(u_int)constmap[i][5]);
8816       if((regs[i].isconst>>6)&1) printf("esi=%x ",(u_int)constmap[i][6]);
8817       if((regs[i].isconst>>7)&1) printf("edi=%x ",(u_int)constmap[i][7]);
8818       #endif
8819       #if defined(__arm__) || defined(__aarch64__)
8820       int r;
8821       for (r = 0; r < ARRAY_SIZE(constmap[i]); r++)
8822         if ((regs[i].isconst >> r) & 1)
8823           printf(" r%d=%x", r, (u_int)constmap[i][r]);
8824       #endif
8825       printf("\n");
8826     }
8827     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
8828       #if defined(__i386__) || defined(__x86_64__)
8829       printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
8830       if(branch_regs[i].dirty&1) printf("eax ");
8831       if((branch_regs[i].dirty>>1)&1) printf("ecx ");
8832       if((branch_regs[i].dirty>>2)&1) printf("edx ");
8833       if((branch_regs[i].dirty>>3)&1) printf("ebx ");
8834       if((branch_regs[i].dirty>>5)&1) printf("ebp ");
8835       if((branch_regs[i].dirty>>6)&1) printf("esi ");
8836       if((branch_regs[i].dirty>>7)&1) printf("edi ");
8837       #endif
8838       #ifdef __arm__
8839       printf("branch(%d): r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[4],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7],branch_regs[i].regmap[8],branch_regs[i].regmap[9],branch_regs[i].regmap[10],branch_regs[i].regmap[12]);
8840       if(branch_regs[i].dirty&1) printf("r0 ");
8841       if((branch_regs[i].dirty>>1)&1) printf("r1 ");
8842       if((branch_regs[i].dirty>>2)&1) printf("r2 ");
8843       if((branch_regs[i].dirty>>3)&1) printf("r3 ");
8844       if((branch_regs[i].dirty>>4)&1) printf("r4 ");
8845       if((branch_regs[i].dirty>>5)&1) printf("r5 ");
8846       if((branch_regs[i].dirty>>6)&1) printf("r6 ");
8847       if((branch_regs[i].dirty>>7)&1) printf("r7 ");
8848       if((branch_regs[i].dirty>>8)&1) printf("r8 ");
8849       if((branch_regs[i].dirty>>9)&1) printf("r9 ");
8850       if((branch_regs[i].dirty>>10)&1) printf("r10 ");
8851       if((branch_regs[i].dirty>>12)&1) printf("r12 ");
8852       #endif
8853     }
8854   }
8855 #endif // DISASM
8856
8857   /* Pass 8 - Assembly */
8858   linkcount=0;stubcount=0;
8859   ds=0;is_delayslot=0;
8860   u_int dirty_pre=0;
8861   void *beginning=start_block();
8862   if((u_int)addr&1) {
8863     ds=1;
8864     pagespan_ds();
8865   }
8866   void *instr_addr0_override = NULL;
8867
8868   if (start == 0x80030000) {
8869     // nasty hack for the fastbios thing
8870     // override block entry to this code
8871     instr_addr0_override = out;
8872     emit_movimm(start,0);
8873     // abuse io address var as a flag that we
8874     // have already returned here once
8875     emit_readword(&address,1);
8876     emit_writeword(0,&pcaddr);
8877     emit_writeword(0,&address);
8878     emit_cmp(0,1);
8879     #ifdef __aarch64__
8880     emit_jeq(out + 4*2);
8881     emit_far_jump(new_dyna_leave);
8882     #else
8883     emit_jne(new_dyna_leave);
8884     #endif
8885   }
8886   for(i=0;i<slen;i++)
8887   {
8888     //if(ds) printf("ds: ");
8889     disassemble_inst(i);
8890     if(ds) {
8891       ds=0; // Skip delay slot
8892       if(bt[i]) assem_debug("OOPS - branch into delay slot\n");
8893       instr_addr[i] = NULL;
8894     } else {
8895       speculate_register_values(i);
8896       #ifndef DESTRUCTIVE_WRITEBACK
8897       if (i < 2 || !is_ujump(i-2))
8898       {
8899         wb_valid(regmap_pre[i],regs[i].regmap_entry,dirty_pre,regs[i].wasdirty,unneeded_reg[i]);
8900       }
8901       if((itype[i]==CJUMP||itype[i]==SJUMP)&&!likely[i]) {
8902         dirty_pre=branch_regs[i].dirty;
8903       }else{
8904         dirty_pre=regs[i].dirty;
8905       }
8906       #endif
8907       // write back
8908       if (i < 2 || !is_ujump(i-2))
8909       {
8910         wb_invalidate(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,unneeded_reg[i]);
8911         loop_preload(regmap_pre[i],regs[i].regmap_entry);
8912       }
8913       // branch target entry point
8914       instr_addr[i] = out;
8915       assem_debug("<->\n");
8916       drc_dbg_emit_do_cmp(i);
8917
8918       // load regs
8919       if(regs[i].regmap_entry[HOST_CCREG]==CCREG&&regs[i].regmap[HOST_CCREG]!=CCREG)
8920         wb_register(CCREG,regs[i].regmap_entry,regs[i].wasdirty);
8921       load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i],rs2[i]);
8922       address_generation(i,&regs[i],regs[i].regmap_entry);
8923       load_consts(regmap_pre[i],regs[i].regmap,i);
8924       if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8925       {
8926         // Load the delay slot registers if necessary
8927         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i]&&(rs1[i+1]!=rt1[i]||rt1[i]==0))
8928           load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8929         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i]&&(rs2[i+1]!=rt1[i]||rt1[i]==0))
8930           load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8931         if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a)
8932           load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8933       }
8934       else if(i+1<slen)
8935       {
8936         // Preload registers for following instruction
8937         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
8938           if(rs1[i+1]!=rt1[i]&&rs1[i+1]!=rt2[i])
8939             load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8940         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
8941           if(rs2[i+1]!=rt1[i]&&rs2[i+1]!=rt2[i])
8942             load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8943       }
8944       // TODO: if(is_ooo(i)) address_generation(i+1);
8945       if(itype[i]==CJUMP)
8946         load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
8947       if(itype[i]==STORE||itype[i]==STORELR||(opcode[i]&0x3b)==0x39||(opcode[i]&0x3b)==0x3a)
8948         load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8949       // assemble
8950       switch(itype[i]) {
8951         case ALU:
8952           alu_assemble(i,&regs[i]);break;
8953         case IMM16:
8954           imm16_assemble(i,&regs[i]);break;
8955         case SHIFT:
8956           shift_assemble(i,&regs[i]);break;
8957         case SHIFTIMM:
8958           shiftimm_assemble(i,&regs[i]);break;
8959         case LOAD:
8960           load_assemble(i,&regs[i]);break;
8961         case LOADLR:
8962           loadlr_assemble(i,&regs[i]);break;
8963         case STORE:
8964           store_assemble(i,&regs[i]);break;
8965         case STORELR:
8966           storelr_assemble(i,&regs[i]);break;
8967         case COP0:
8968           cop0_assemble(i,&regs[i]);break;
8969         case COP1:
8970           cop1_assemble(i,&regs[i]);break;
8971         case C1LS:
8972           c1ls_assemble(i,&regs[i]);break;
8973         case COP2:
8974           cop2_assemble(i,&regs[i]);break;
8975         case C2LS:
8976           c2ls_assemble(i,&regs[i]);break;
8977         case C2OP:
8978           c2op_assemble(i,&regs[i]);break;
8979         case MULTDIV:
8980           multdiv_assemble(i,&regs[i]);break;
8981         case MOV:
8982           mov_assemble(i,&regs[i]);break;
8983         case SYSCALL:
8984           syscall_assemble(i,&regs[i]);break;
8985         case HLECALL:
8986           hlecall_assemble(i,&regs[i]);break;
8987         case INTCALL:
8988           intcall_assemble(i,&regs[i]);break;
8989         case UJUMP:
8990           ujump_assemble(i,&regs[i]);ds=1;break;
8991         case RJUMP:
8992           rjump_assemble(i,&regs[i]);ds=1;break;
8993         case CJUMP:
8994           cjump_assemble(i,&regs[i]);ds=1;break;
8995         case SJUMP:
8996           sjump_assemble(i,&regs[i]);ds=1;break;
8997         case SPAN:
8998           pagespan_assemble(i,&regs[i]);break;
8999       }
9000       if (is_ujump(i))
9001         literal_pool(1024);
9002       else
9003         literal_pool_jumpover(256);
9004     }
9005   }
9006   //assert(is_ujump(i-2));
9007   // If the block did not end with an unconditional branch,
9008   // add a jump to the next instruction.
9009   if(i>1) {
9010     if(!is_ujump(i-2)&&itype[i-1]!=SPAN) {
9011       assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
9012       assert(i==slen);
9013       if(itype[i-2]!=CJUMP&&itype[i-2]!=SJUMP) {
9014         store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
9015         if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
9016           emit_loadreg(CCREG,HOST_CCREG);
9017         emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
9018       }
9019       else if(!likely[i-2])
9020       {
9021         store_regs_bt(branch_regs[i-2].regmap,branch_regs[i-2].dirty,start+i*4);
9022         assert(branch_regs[i-2].regmap[HOST_CCREG]==CCREG);
9023       }
9024       else
9025       {
9026         store_regs_bt(regs[i-2].regmap,regs[i-2].dirty,start+i*4);
9027         assert(regs[i-2].regmap[HOST_CCREG]==CCREG);
9028       }
9029       add_to_linker(out,start+i*4,0);
9030       emit_jmp(0);
9031     }
9032   }
9033   else
9034   {
9035     assert(i>0);
9036     assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
9037     store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
9038     if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
9039       emit_loadreg(CCREG,HOST_CCREG);
9040     emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
9041     add_to_linker(out,start+i*4,0);
9042     emit_jmp(0);
9043   }
9044
9045   // TODO: delay slot stubs?
9046   // Stubs
9047   for(i=0;i<stubcount;i++)
9048   {
9049     switch(stubs[i].type)
9050     {
9051       case LOADB_STUB:
9052       case LOADH_STUB:
9053       case LOADW_STUB:
9054       case LOADD_STUB:
9055       case LOADBU_STUB:
9056       case LOADHU_STUB:
9057         do_readstub(i);break;
9058       case STOREB_STUB:
9059       case STOREH_STUB:
9060       case STOREW_STUB:
9061       case STORED_STUB:
9062         do_writestub(i);break;
9063       case CC_STUB:
9064         do_ccstub(i);break;
9065       case INVCODE_STUB:
9066         do_invstub(i);break;
9067       case FP_STUB:
9068         do_cop1stub(i);break;
9069       case STORELR_STUB:
9070         do_unalignedwritestub(i);break;
9071     }
9072   }
9073
9074   if (instr_addr0_override)
9075     instr_addr[0] = instr_addr0_override;
9076
9077   /* Pass 9 - Linker */
9078   for(i=0;i<linkcount;i++)
9079   {
9080     assem_debug("%p -> %8x\n",link_addr[i].addr,link_addr[i].target);
9081     literal_pool(64);
9082     if (!link_addr[i].ext)
9083     {
9084       void *stub = out;
9085       void *addr = check_addr(link_addr[i].target);
9086       emit_extjump(link_addr[i].addr, link_addr[i].target);
9087       if (addr) {
9088         set_jump_target(link_addr[i].addr, addr);
9089         add_link(link_addr[i].target,stub);
9090       }
9091       else
9092         set_jump_target(link_addr[i].addr, stub);
9093     }
9094     else
9095     {
9096       // Internal branch
9097       int target=(link_addr[i].target-start)>>2;
9098       assert(target>=0&&target<slen);
9099       assert(instr_addr[target]);
9100       //#ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
9101       //set_jump_target_fillslot(link_addr[i].addr,instr_addr[target],link_addr[i].ext>>1);
9102       //#else
9103       set_jump_target(link_addr[i].addr, instr_addr[target]);
9104       //#endif
9105     }
9106   }
9107   // External Branch Targets (jump_in)
9108   if(copy+slen*4>(void *)shadow+sizeof(shadow)) copy=shadow;
9109   for(i=0;i<slen;i++)
9110   {
9111     if(bt[i]||i==0)
9112     {
9113       if(instr_addr[i]) // TODO - delay slots (=null)
9114       {
9115         u_int vaddr=start+i*4;
9116         u_int page=get_page(vaddr);
9117         u_int vpage=get_vpage(vaddr);
9118         literal_pool(256);
9119         {
9120           assem_debug("%p (%d) <- %8x\n",instr_addr[i],i,start+i*4);
9121           assem_debug("jump_in: %x\n",start+i*4);
9122           ll_add(jump_dirty+vpage,vaddr,out);
9123           void *entry_point = do_dirty_stub(i);
9124           ll_add_flags(jump_in+page,vaddr,state_rflags,entry_point);
9125           // If there was an existing entry in the hash table,
9126           // replace it with the new address.
9127           // Don't add new entries.  We'll insert the
9128           // ones that actually get used in check_addr().
9129           struct ht_entry *ht_bin = hash_table_get(vaddr);
9130           if (ht_bin->vaddr[0] == vaddr)
9131             ht_bin->tcaddr[0] = entry_point;
9132           if (ht_bin->vaddr[1] == vaddr)
9133             ht_bin->tcaddr[1] = entry_point;
9134         }
9135       }
9136     }
9137   }
9138   // Write out the literal pool if necessary
9139   literal_pool(0);
9140   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
9141   // Align code
9142   if(((u_int)out)&7) emit_addnop(13);
9143   #endif
9144   assert(out - (u_char *)beginning < MAX_OUTPUT_BLOCK_SIZE);
9145   //printf("shadow buffer: %p-%p\n",copy,(u_char *)copy+slen*4);
9146   memcpy(copy,source,slen*4);
9147   copy+=slen*4;
9148
9149   end_block(beginning);
9150
9151   // If we're within 256K of the end of the buffer,
9152   // start over from the beginning. (Is 256K enough?)
9153   if (out > ndrc->translation_cache + sizeof(ndrc->translation_cache) - MAX_OUTPUT_BLOCK_SIZE)
9154     out = ndrc->translation_cache;
9155
9156   // Trap writes to any of the pages we compiled
9157   for(i=start>>12;i<=(start+slen*4)>>12;i++) {
9158     invalid_code[i]=0;
9159   }
9160   inv_code_start=inv_code_end=~0;
9161
9162   // for PCSX we need to mark all mirrors too
9163   if(get_page(start)<(RAM_SIZE>>12))
9164     for(i=start>>12;i<=(start+slen*4)>>12;i++)
9165       invalid_code[((u_int)0x00000000>>12)|(i&0x1ff)]=
9166       invalid_code[((u_int)0x80000000>>12)|(i&0x1ff)]=
9167       invalid_code[((u_int)0xa0000000>>12)|(i&0x1ff)]=0;
9168
9169   /* Pass 10 - Free memory by expiring oldest blocks */
9170
9171   int end=(((out-ndrc->translation_cache)>>(TARGET_SIZE_2-16))+16384)&65535;
9172   while(expirep!=end)
9173   {
9174     int shift=TARGET_SIZE_2-3; // Divide into 8 blocks
9175     uintptr_t base=(uintptr_t)ndrc->translation_cache+((expirep>>13)<<shift); // Base address of this block
9176     inv_debug("EXP: Phase %d\n",expirep);
9177     switch((expirep>>11)&3)
9178     {
9179       case 0:
9180         // Clear jump_in and jump_dirty
9181         ll_remove_matching_addrs(jump_in+(expirep&2047),base,shift);
9182         ll_remove_matching_addrs(jump_dirty+(expirep&2047),base,shift);
9183         ll_remove_matching_addrs(jump_in+2048+(expirep&2047),base,shift);
9184         ll_remove_matching_addrs(jump_dirty+2048+(expirep&2047),base,shift);
9185         break;
9186       case 1:
9187         // Clear pointers
9188         ll_kill_pointers(jump_out[expirep&2047],base,shift);
9189         ll_kill_pointers(jump_out[(expirep&2047)+2048],base,shift);
9190         break;
9191       case 2:
9192         // Clear hash table
9193         for(i=0;i<32;i++) {
9194           struct ht_entry *ht_bin = &hash_table[((expirep&2047)<<5)+i];
9195           if (((uintptr_t)ht_bin->tcaddr[1]>>shift) == (base>>shift) ||
9196              (((uintptr_t)ht_bin->tcaddr[1]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
9197             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[1],ht_bin->tcaddr[1]);
9198             ht_bin->vaddr[1] = -1;
9199             ht_bin->tcaddr[1] = NULL;
9200           }
9201           if (((uintptr_t)ht_bin->tcaddr[0]>>shift) == (base>>shift) ||
9202              (((uintptr_t)ht_bin->tcaddr[0]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
9203             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[0],ht_bin->tcaddr[0]);
9204             ht_bin->vaddr[0] = ht_bin->vaddr[1];
9205             ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
9206             ht_bin->vaddr[1] = -1;
9207             ht_bin->tcaddr[1] = NULL;
9208           }
9209         }
9210         break;
9211       case 3:
9212         // Clear jump_out
9213         if((expirep&2047)==0)
9214           do_clear_cache();
9215         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
9216         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
9217         break;
9218     }
9219     expirep=(expirep+1)&65535;
9220   }
9221   return 0;
9222 }
9223
9224 // vim:shiftwidth=2:expandtab