libpcsxcore/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2011 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdlib.h>
  22 #include <stdint.h> //include for uint64_t
  23 #include <assert.h>
  24 #include <errno.h>
  25 #include <sys/mman.h>
  26 #ifdef __MACH__
  27 #include <libkern/OSCacheControl.h>
  28 #endif
  29 #ifdef _3DS
  30 #include <3ds_utils.h>
  31 #endif
  32 #ifdef VITA
  33 #include <psp2/kernel/sysmem.h>
  34 static int sceBlock;
  35 #endif
  36
  37 #include "new_dynarec_config.h"
  38 #include "../psxhle.h"
  39 #include "../psxinterpreter.h"
  40 #include "emu_if.h" //emulator interface
  41
  42 #define noinline __attribute__((noinline,noclone))
  43 #ifndef ARRAY_SIZE
  44 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
  45 #endif
  46
  47 //#define DISASM
  48 //#define assem_debug printf
  49 //#define inv_debug printf
  50 #define assem_debug(...)
  51 #define inv_debug(...)
  52
  53 #ifdef __i386__
  54 #include "assem_x86.h"
  55 #endif
  56 #ifdef __x86_64__
  57 #include "assem_x64.h"
  58 #endif
  59 #ifdef __arm__
  60 #include "assem_arm.h"
  61 #endif
  62 #ifdef __aarch64__
  63 #include "assem_arm64.h"
  64 #endif
  65
  66 #define MAXBLOCK 4096
  67 #define MAX_OUTPUT_BLOCK_SIZE 262144
  68
  69 struct ndrc_mem
  70 {
  71   u_char translation_cache[1 << TARGET_SIZE_2];
  72   struct
  73   {
  74     struct tramp_insns ops[2048 / sizeof(struct tramp_insns)];
  75     const void *f[2048 / sizeof(void *)];
  76   } tramp;
  77 };
  78
  79 #ifdef BASE_ADDR_DYNAMIC
  80 static struct ndrc_mem *ndrc;
  81 #else
  82 static struct ndrc_mem ndrc_ __attribute__((aligned(4096)));
  83 static struct ndrc_mem *ndrc = &ndrc_;
  84 #endif
  85
  86 // stubs
  87 enum stub_type {
  88   CC_STUB = 1,
  89   FP_STUB = 2,
  90   LOADB_STUB = 3,
  91   LOADH_STUB = 4,
  92   LOADW_STUB = 5,
  93   LOADD_STUB = 6,
  94   LOADBU_STUB = 7,
  95   LOADHU_STUB = 8,
  96   STOREB_STUB = 9,
  97   STOREH_STUB = 10,
  98   STOREW_STUB = 11,
  99   STORED_STUB = 12,
 100   STORELR_STUB = 13,
 101   INVCODE_STUB = 14,
 102 };
 103
 104 struct regstat
 105 {
 106   signed char regmap_entry[HOST_REGS];
 107   signed char regmap[HOST_REGS];
 108   uint64_t wasdirty;
 109   uint64_t dirty;
 110   uint64_t u;
 111   u_int wasconst;
 112   u_int isconst;
 113   u_int loadedconst;             // host regs that have constants loaded
 114   u_int waswritten;              // MIPS regs that were used as store base before
 115 };
 116
 117 // note: asm depends on this layout
 118 struct ll_entry
 119 {
 120   u_int vaddr;
 121   u_int reg_sv_flags;
 122   void *addr;
 123   struct ll_entry *next;
 124 };
 125
 126 struct ht_entry
 127 {
 128   u_int vaddr[2];
 129   void *tcaddr[2];
 130 };
 131
 132 struct code_stub
 133 {
 134   enum stub_type type;
 135   void *addr;
 136   void *retaddr;
 137   u_int a;
 138   uintptr_t b;
 139   uintptr_t c;
 140   u_int d;
 141   u_int e;
 142 };
 143
 144 struct link_entry
 145 {
 146   void *addr;
 147   u_int target;
 148   u_int ext;
 149 };
 150
 151   // used by asm:
 152   u_char *out;
 153   struct ht_entry hash_table[65536]  __attribute__((aligned(16)));
 154   struct ll_entry *jump_in[4096] __attribute__((aligned(16)));
 155   struct ll_entry *jump_dirty[4096];
 156
 157   static struct ll_entry *jump_out[4096];
 158   static u_int start;
 159   static u_int *source;
 160   static char insn[MAXBLOCK][10];
 161   static u_char itype[MAXBLOCK];
 162   static u_char opcode[MAXBLOCK];
 163   static u_char opcode2[MAXBLOCK];
 164   static u_char bt[MAXBLOCK];
 165   static u_char rs1[MAXBLOCK];
 166   static u_char rs2[MAXBLOCK];
 167   static u_char rt1[MAXBLOCK];
 168   static u_char rt2[MAXBLOCK];
 169   static u_char dep1[MAXBLOCK];
 170   static u_char dep2[MAXBLOCK];
 171   static u_char lt1[MAXBLOCK];
 172   static uint64_t gte_rs[MAXBLOCK]; // gte: 32 data and 32 ctl regs
 173   static uint64_t gte_rt[MAXBLOCK];
 174   static uint64_t gte_unneeded[MAXBLOCK];
 175   static u_int smrv[32]; // speculated MIPS register values
 176   static u_int smrv_strong; // mask or regs that are likely to have correct values
 177   static u_int smrv_weak; // same, but somewhat less likely
 178   static u_int smrv_strong_next; // same, but after current insn executes
 179   static u_int smrv_weak_next;
 180   static int imm[MAXBLOCK];
 181   static u_int ba[MAXBLOCK];
 182   static char likely[MAXBLOCK];
 183   static char is_ds[MAXBLOCK];
 184   static char ooo[MAXBLOCK];
 185   static uint64_t unneeded_reg[MAXBLOCK];
 186   static uint64_t branch_unneeded_reg[MAXBLOCK];
 187   static signed char regmap_pre[MAXBLOCK][HOST_REGS]; // pre-instruction i?
 188   static uint64_t current_constmap[HOST_REGS];
 189   static uint64_t constmap[MAXBLOCK][HOST_REGS];
 190   static struct regstat regs[MAXBLOCK];
 191   static struct regstat branch_regs[MAXBLOCK];
 192   static signed char minimum_free_regs[MAXBLOCK];
 193   static u_int needed_reg[MAXBLOCK];
 194   static u_int wont_dirty[MAXBLOCK];
 195   static u_int will_dirty[MAXBLOCK];
 196   static int ccadj[MAXBLOCK];
 197   static int slen;
 198   static void *instr_addr[MAXBLOCK];
 199   static struct link_entry link_addr[MAXBLOCK];
 200   static int linkcount;
 201   static struct code_stub stubs[MAXBLOCK*3];
 202   static int stubcount;
 203   static u_int literals[1024][2];
 204   static int literalcount;
 205   static int is_delayslot;
 206   static char shadow[1048576]  __attribute__((aligned(16)));
 207   static void *copy;
 208   static int expirep;
 209   static u_int stop_after_jal;
 210 #ifndef RAM_FIXED
 211   static uintptr_t ram_offset;
 212 #else
 213   static const uintptr_t ram_offset=0;
 214 #endif
 215
 216   int new_dynarec_hacks;
 217   int new_dynarec_did_compile;
 218
 219   extern int cycle_count; // ... until end of the timeslice, counts -N -> 0
 220   extern int last_count;  // last absolute target, often = next_interupt
 221   extern int pcaddr;
 222   extern int pending_exception;
 223   extern int branch_target;
 224   extern uintptr_t mini_ht[32][2];
 225   extern u_char restore_candidate[512];
 226
 227   /* registers that may be allocated */
 228   /* 1-31 gpr */
 229 #define LOREG 32 // lo
 230 #define HIREG 33 // hi
 231 //#define FSREG 34 // FPU status (FCSR)
 232 #define CSREG 35 // Coprocessor status
 233 #define CCREG 36 // Cycle count
 234 #define INVCP 37 // Pointer to invalid_code
 235 //#define MMREG 38 // Pointer to memory_map
 236 //#define ROREG 39 // ram offset (if rdram!=0x80000000)
 237 #define TEMPREG 40
 238 #define FTEMP 40 // FPU temporary register
 239 #define PTEMP 41 // Prefetch temporary register
 240 //#define TLREG 42 // TLB mapping offset
 241 #define RHASH 43 // Return address hash
 242 #define RHTBL 44 // Return address hash table address
 243 #define RTEMP 45 // JR/JALR address register
 244 #define MAXREG 45
 245 #define AGEN1 46 // Address generation temporary register
 246 //#define AGEN2 47 // Address generation temporary register
 247 //#define MGEN1 48 // Maptable address generation temporary register
 248 //#define MGEN2 49 // Maptable address generation temporary register
 249 #define BTREG 50 // Branch target temporary register
 250
 251   /* instruction types */
 252 #define NOP 0     // No operation
 253 #define LOAD 1    // Load
 254 #define STORE 2   // Store
 255 #define LOADLR 3  // Unaligned load
 256 #define STORELR 4 // Unaligned store
 257 #define MOV 5     // Move
 258 #define ALU 6     // Arithmetic/logic
 259 #define MULTDIV 7 // Multiply/divide
 260 #define SHIFT 8   // Shift by register
 261 #define SHIFTIMM 9// Shift by immediate
 262 #define IMM16 10  // 16-bit immediate
 263 #define RJUMP 11  // Unconditional jump to register
 264 #define UJUMP 12  // Unconditional jump
 265 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 266 #define SJUMP 14  // Conditional branch (regimm format)
 267 #define COP0 15   // Coprocessor 0
 268 #define COP1 16   // Coprocessor 1
 269 #define C1LS 17   // Coprocessor 1 load/store
 270 //#define FJUMP 18  // Conditional branch (floating point)
 271 //#define FLOAT 19  // Floating point unit
 272 //#define FCONV 20  // Convert integer to float
 273 //#define FCOMP 21  // Floating point compare (sets FSREG)
 274 #define SYSCALL 22// SYSCALL
 275 #define OTHER 23  // Other
 276 #define SPAN 24   // Branch/delay slot spans 2 pages
 277 #define NI 25     // Not implemented
 278 #define HLECALL 26// PCSX fake opcodes for HLE
 279 #define COP2 27   // Coprocessor 2 move
 280 #define C2LS 28   // Coprocessor 2 load/store
 281 #define C2OP 29   // Coprocessor 2 operation
 282 #define INTCALL 30// Call interpreter to handle rare corner cases
 283
 284   /* branch codes */
 285 #define TAKEN 1
 286 #define NOTTAKEN 2
 287 #define NULLDS 3
 288
 289 #define DJT_1 (void *)1l // no function, just a label in assem_debug log
 290 #define DJT_2 (void *)2l
 291
 292 // asm linkage
 293 int new_recompile_block(u_int addr);
 294 void *get_addr_ht(u_int vaddr);
 295 void invalidate_block(u_int block);
 296 void invalidate_addr(u_int addr);
 297 void remove_hash(int vaddr);
 298 void dyna_linker();
 299 void dyna_linker_ds();
 300 void verify_code();
 301 void verify_code_ds();
 302 void cc_interrupt();
 303 void fp_exception();
 304 void fp_exception_ds();
 305 void jump_to_new_pc();
 306 void new_dyna_leave();
 307
 308 // Needed by assembler
 309 static void wb_register(signed char r,signed char regmap[],uint64_t dirty);
 310 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty);
 311 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr);
 312 static void load_all_regs(signed char i_regmap[]);
 313 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 314 static void load_regs_entry(int t);
 315 static void load_all_consts(signed char regmap[],u_int dirty,int i);
 316
 317 static int verify_dirty(const u_int *ptr);
 318 static int get_final_value(int hr, int i, int *value);
 319 static void add_stub(enum stub_type type, void *addr, void *retaddr,
 320   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e);
 321 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
 322   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist);
 323 static void add_to_linker(void *addr, u_int target, int ext);
 324 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override);
 325 static void *get_direct_memhandler(void *table, u_int addr,
 326   enum stub_type type, uintptr_t *addr_host);
 327 static void pass_args(int a0, int a1);
 328 static void emit_far_jump(const void *f);
 329 static void emit_far_call(const void *f);
 330
 331 static void mprotect_w_x(void *start, void *end, int is_x)
 332 {
 333 #ifdef NO_WRITE_EXEC
 334   #if defined(VITA)
 335   // *Open* enables write on all memory that was
 336   // allocated by sceKernelAllocMemBlockForVM()?
 337   if (is_x)
 338     sceKernelCloseVMDomain();
 339   else
 340     sceKernelOpenVMDomain();
 341   #else
 342   u_long mstart = (u_long)start & ~4095ul;
 343   u_long mend = (u_long)end;
 344   if (mprotect((void *)mstart, mend - mstart,
 345                PROT_READ | (is_x ? PROT_EXEC : PROT_WRITE)) != 0)
 346     SysPrintf("mprotect(%c) failed: %s\n", is_x ? 'x' : 'w', strerror(errno));
 347   #endif
 348 #endif
 349 }
 350
 351 static void start_tcache_write(void *start, void *end)
 352 {
 353   mprotect_w_x(start, end, 0);
 354 }
 355
 356 static void end_tcache_write(void *start, void *end)
 357 {
 358 #if defined(__arm__) || defined(__aarch64__)
 359   size_t len = (char *)end - (char *)start;
 360   #if   defined(__BLACKBERRY_QNX__)
 361   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);
 362   #elif defined(__MACH__)
 363   sys_cache_control(kCacheFunctionPrepareForExecution, start, len);
 364   #elif defined(VITA)
 365   sceKernelSyncVMDomain(sceBlock, start, len);
 366   #elif defined(_3DS)
 367   ctr_flush_invalidate_cache();
 368   #elif defined(__aarch64__)
 369   // as of 2021, __clear_cache() is still broken on arm64
 370   // so here is a custom one :(
 371   clear_cache_arm64(start, end);
 372   #else
 373   __clear_cache(start, end);
 374   #endif
 375   (void)len;
 376 #endif
 377
 378   mprotect_w_x(start, end, 1);
 379 }
 380
 381 static void *start_block(void)
 382 {
 383   u_char *end = out + MAX_OUTPUT_BLOCK_SIZE;
 384   if (end > ndrc->translation_cache + sizeof(ndrc->translation_cache))
 385     end = ndrc->translation_cache + sizeof(ndrc->translation_cache);
 386   start_tcache_write(out, end);
 387   return out;
 388 }
 389
 390 static void end_block(void *start)
 391 {
 392   end_tcache_write(start, out);
 393 }
 394
 395 // also takes care of w^x mappings when patching code
 396 static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)];
 397
 398 static void mark_clear_cache(void *target)
 399 {
 400   uintptr_t offset = (u_char *)target - ndrc->translation_cache;
 401   u_int mask = 1u << ((offset >> 12) & 31);
 402   if (!(needs_clear_cache[offset >> 17] & mask)) {
 403     char *start = (char *)((uintptr_t)target & ~4095l);
 404     start_tcache_write(start, start + 4095);
 405     needs_clear_cache[offset >> 17] |= mask;
 406   }
 407 }
 408
 409 // Clearing the cache is rather slow on ARM Linux, so mark the areas
 410 // that need to be cleared, and then only clear these areas once.
 411 static void do_clear_cache(void)
 412 {
 413   int i, j;
 414   for (i = 0; i < (1<<(TARGET_SIZE_2-17)); i++)
 415   {
 416     u_int bitmap = needs_clear_cache[i];
 417     if (!bitmap)
 418       continue;
 419     for (j = 0; j < 32; j++)
 420     {
 421       u_char *start, *end;
 422       if (!(bitmap & (1<<j)))
 423         continue;
 424
 425       start = ndrc->translation_cache + i*131072 + j*4096;
 426       end = start + 4095;
 427       for (j++; j < 32; j++) {
 428         if (!(bitmap & (1<<j)))
 429           break;
 430         end += 4096;
 431       }
 432       end_tcache_write(start, end);
 433     }
 434     needs_clear_cache[i] = 0;
 435   }
 436 }
 437
 438 //#define DEBUG_CYCLE_COUNT 1
 439
 440 #define NO_CYCLE_PENALTY_THR 12
 441
 442 int cycle_multiplier; // 100 for 1.0
 443
 444 static int CLOCK_ADJUST(int x)
 445 {
 446   int s=(x>>31)|1;
 447   return (x * cycle_multiplier + s * 50) / 100;
 448 }
 449
 450 static u_int get_page(u_int vaddr)
 451 {
 452   u_int page=vaddr&~0xe0000000;
 453   if (page < 0x1000000)
 454     page &= ~0x0e00000; // RAM mirrors
 455   page>>=12;
 456   if(page>2048) page=2048+(page&2047);
 457   return page;
 458 }
 459
 460 // no virtual mem in PCSX
 461 static u_int get_vpage(u_int vaddr)
 462 {
 463   return get_page(vaddr);
 464 }
 465
 466 static struct ht_entry *hash_table_get(u_int vaddr)
 467 {
 468   return &hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 469 }
 470
 471 static void hash_table_add(struct ht_entry *ht_bin, u_int vaddr, void *tcaddr)
 472 {
 473   ht_bin->vaddr[1] = ht_bin->vaddr[0];
 474   ht_bin->tcaddr[1] = ht_bin->tcaddr[0];
 475   ht_bin->vaddr[0] = vaddr;
 476   ht_bin->tcaddr[0] = tcaddr;
 477 }
 478
 479 // some messy ari64's code, seems to rely on unsigned 32bit overflow
 480 static int doesnt_expire_soon(void *tcaddr)
 481 {
 482   u_int diff = (u_int)((u_char *)tcaddr - out) << (32-TARGET_SIZE_2);
 483   return diff > (u_int)(0x60000000 + (MAX_OUTPUT_BLOCK_SIZE << (32-TARGET_SIZE_2)));
 484 }
 485
 486 // Get address from virtual address
 487 // This is called from the recompiled JR/JALR instructions
 488 void noinline *get_addr(u_int vaddr)
 489 {
 490   u_int page=get_page(vaddr);
 491   u_int vpage=get_vpage(vaddr);
 492   struct ll_entry *head;
 493   //printf("TRACE: count=%d next=%d (get_addr %x,page %d)\n",Count,next_interupt,vaddr,page);
 494   head=jump_in[page];
 495   while(head!=NULL) {
 496     if(head->vaddr==vaddr) {
 497   //printf("TRACE: count=%d next=%d (get_addr match %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 498       hash_table_add(hash_table_get(vaddr), vaddr, head->addr);
 499       return head->addr;
 500     }
 501     head=head->next;
 502   }
 503   head=jump_dirty[vpage];
 504   while(head!=NULL) {
 505     if(head->vaddr==vaddr) {
 506       //printf("TRACE: count=%d next=%d (get_addr match dirty %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 507       // Don't restore blocks which are about to expire from the cache
 508       if (doesnt_expire_soon(head->addr))
 509       if (verify_dirty(head->addr)) {
 510         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 511         invalid_code[vaddr>>12]=0;
 512         inv_code_start=inv_code_end=~0;
 513         if(vpage<2048) {
 514           restore_candidate[vpage>>3]|=1<<(vpage&7);
 515         }
 516         else restore_candidate[page>>3]|=1<<(page&7);
 517         struct ht_entry *ht_bin = hash_table_get(vaddr);
 518         if (ht_bin->vaddr[0] == vaddr)
 519           ht_bin->tcaddr[0] = head->addr; // Replace existing entry
 520         else
 521           hash_table_add(ht_bin, vaddr, head->addr);
 522
 523         return head->addr;
 524       }
 525     }
 526     head=head->next;
 527   }
 528   //printf("TRACE: count=%d next=%d (get_addr no-match %x)\n",Count,next_interupt,vaddr);
 529   int r=new_recompile_block(vaddr);
 530   if(r==0) return get_addr(vaddr);
 531   // Execute in unmapped page, generate pagefault execption
 532   Status|=2;
 533   Cause=(vaddr<<31)|0x8;
 534   EPC=(vaddr&1)?vaddr-5:vaddr;
 535   BadVAddr=(vaddr&~1);
 536   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 537   EntryHi=BadVAddr&0xFFFFE000;
 538   return get_addr_ht(0x80000000);
 539 }
 540 // Look up address in hash table first
 541 void *get_addr_ht(u_int vaddr)
 542 {
 543   //printf("TRACE: count=%d next=%d (get_addr_ht %x)\n",Count,next_interupt,vaddr);
 544   const struct ht_entry *ht_bin = hash_table_get(vaddr);
 545   if (ht_bin->vaddr[0] == vaddr) return ht_bin->tcaddr[0];
 546   if (ht_bin->vaddr[1] == vaddr) return ht_bin->tcaddr[1];
 547   return get_addr(vaddr);
 548 }
 549
 550 void clear_all_regs(signed char regmap[])
 551 {
 552   int hr;
 553   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 554 }
 555
 556 static signed char get_reg(const signed char regmap[],int r)
 557 {
 558   int hr;
 559   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 560   return -1;
 561 }
 562
 563 // Find a register that is available for two consecutive cycles
 564 static signed char get_reg2(signed char regmap1[], const signed char regmap2[], int r)
 565 {
 566   int hr;
 567   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 568   return -1;
 569 }
 570
 571 int count_free_regs(signed char regmap[])
 572 {
 573   int count=0;
 574   int hr;
 575   for(hr=0;hr<HOST_REGS;hr++)
 576   {
 577     if(hr!=EXCLUDE_REG) {
 578       if(regmap[hr]<0) count++;
 579     }
 580   }
 581   return count;
 582 }
 583
 584 void dirty_reg(struct regstat *cur,signed char reg)
 585 {
 586   int hr;
 587   if(!reg) return;
 588   for (hr=0;hr<HOST_REGS;hr++) {
 589     if((cur->regmap[hr]&63)==reg) {
 590       cur->dirty|=1<<hr;
 591     }
 592   }
 593 }
 594
 595 void set_const(struct regstat *cur,signed char reg,uint64_t value)
 596 {
 597   int hr;
 598   if(!reg) return;
 599   for (hr=0;hr<HOST_REGS;hr++) {
 600     if(cur->regmap[hr]==reg) {
 601       cur->isconst|=1<<hr;
 602       current_constmap[hr]=value;
 603     }
 604   }
 605 }
 606
 607 void clear_const(struct regstat *cur,signed char reg)
 608 {
 609   int hr;
 610   if(!reg) return;
 611   for (hr=0;hr<HOST_REGS;hr++) {
 612     if((cur->regmap[hr]&63)==reg) {
 613       cur->isconst&=~(1<<hr);
 614     }
 615   }
 616 }
 617
 618 int is_const(struct regstat *cur,signed char reg)
 619 {
 620   int hr;
 621   if(reg<0) return 0;
 622   if(!reg) return 1;
 623   for (hr=0;hr<HOST_REGS;hr++) {
 624     if((cur->regmap[hr]&63)==reg) {
 625       return (cur->isconst>>hr)&1;
 626     }
 627   }
 628   return 0;
 629 }
 630 uint64_t get_const(struct regstat *cur,signed char reg)
 631 {
 632   int hr;
 633   if(!reg) return 0;
 634   for (hr=0;hr<HOST_REGS;hr++) {
 635     if(cur->regmap[hr]==reg) {
 636       return current_constmap[hr];
 637     }
 638   }
 639   SysPrintf("Unknown constant in r%d\n",reg);
 640   abort();
 641 }
 642
 643 // Least soon needed registers
 644 // Look at the next ten instructions and see which registers
 645 // will be used.  Try not to reallocate these.
 646 void lsn(u_char hsn[], int i, int *preferred_reg)
 647 {
 648   int j;
 649   int b=-1;
 650   for(j=0;j<9;j++)
 651   {
 652     if(i+j>=slen) {
 653       j=slen-i-1;
 654       break;
 655     }
 656     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 657     {
 658       // Don't go past an unconditonal jump
 659       j++;
 660       break;
 661     }
 662   }
 663   for(;j>=0;j--)
 664   {
 665     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 666     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 667     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 668     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 669     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 670       // Stores can allocate zero
 671       hsn[rs1[i+j]]=j;
 672       hsn[rs2[i+j]]=j;
 673     }
 674     // On some architectures stores need invc_ptr
 675     #if defined(HOST_IMM8)
 676     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39 || (opcode[i+j]&0x3b)==0x3a) {
 677       hsn[INVCP]=j;
 678     }
 679     #endif
 680     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 681     {
 682       hsn[CCREG]=j;
 683       b=j;
 684     }
 685   }
 686   if(b>=0)
 687   {
 688     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 689     {
 690       // Follow first branch
 691       int t=(ba[i+b]-start)>>2;
 692       j=7-b;if(t+j>=slen) j=slen-t-1;
 693       for(;j>=0;j--)
 694       {
 695         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 696         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 697         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 698         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 699       }
 700     }
 701     // TODO: preferred register based on backward branch
 702   }
 703   // Delay slot should preferably not overwrite branch conditions or cycle count
 704   if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
 705     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 706     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 707     hsn[CCREG]=1;
 708     // ...or hash tables
 709     hsn[RHASH]=1;
 710     hsn[RHTBL]=1;
 711   }
 712   // Coprocessor load/store needs FTEMP, even if not declared
 713   if(itype[i]==C1LS||itype[i]==C2LS) {
 714     hsn[FTEMP]=0;
 715   }
 716   // Load L/R also uses FTEMP as a temporary register
 717   if(itype[i]==LOADLR) {
 718     hsn[FTEMP]=0;
 719   }
 720   // Also SWL/SWR/SDL/SDR
 721   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) {
 722     hsn[FTEMP]=0;
 723   }
 724   // Don't remove the miniht registers
 725   if(itype[i]==UJUMP||itype[i]==RJUMP)
 726   {
 727     hsn[RHASH]=0;
 728     hsn[RHTBL]=0;
 729   }
 730 }
 731
 732 // We only want to allocate registers if we're going to use them again soon
 733 int needed_again(int r, int i)
 734 {
 735   int j;
 736   int b=-1;
 737   int rn=10;
 738
 739   if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000))
 740   {
 741     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 742       return 0; // Don't need any registers if exiting the block
 743   }
 744   for(j=0;j<9;j++)
 745   {
 746     if(i+j>=slen) {
 747       j=slen-i-1;
 748       break;
 749     }
 750     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 751     {
 752       // Don't go past an unconditonal jump
 753       j++;
 754       break;
 755     }
 756     if(itype[i+j]==SYSCALL||itype[i+j]==HLECALL||itype[i+j]==INTCALL||((source[i+j]&0xfc00003f)==0x0d))
 757     {
 758       break;
 759     }
 760   }
 761   for(;j>=1;j--)
 762   {
 763     if(rs1[i+j]==r) rn=j;
 764     if(rs2[i+j]==r) rn=j;
 765     if((unneeded_reg[i+j]>>r)&1) rn=10;
 766     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 767     {
 768       b=j;
 769     }
 770   }
 771   /*
 772   if(b>=0)
 773   {
 774     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 775     {
 776       // Follow first branch
 777       int o=rn;
 778       int t=(ba[i+b]-start)>>2;
 779       j=7-b;if(t+j>=slen) j=slen-t-1;
 780       for(;j>=0;j--)
 781       {
 782         if(!((unneeded_reg[t+j]>>r)&1)) {
 783           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 784           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 785         }
 786         else rn=o;
 787       }
 788     }
 789   }*/
 790   if(rn<10) return 1;
 791   (void)b;
 792   return 0;
 793 }
 794
 795 // Try to match register allocations at the end of a loop with those
 796 // at the beginning
 797 int loop_reg(int i, int r, int hr)
 798 {
 799   int j,k;
 800   for(j=0;j<9;j++)
 801   {
 802     if(i+j>=slen) {
 803       j=slen-i-1;
 804       break;
 805     }
 806     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 807     {
 808       // Don't go past an unconditonal jump
 809       j++;
 810       break;
 811     }
 812   }
 813   k=0;
 814   if(i>0){
 815     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)
 816       k--;
 817   }
 818   for(;k<j;k++)
 819   {
 820     assert(r < 64);
 821     if((unneeded_reg[i+k]>>r)&1) return hr;
 822     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP))
 823     {
 824       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 825       {
 826         int t=(ba[i+k]-start)>>2;
 827         int reg=get_reg(regs[t].regmap_entry,r);
 828         if(reg>=0) return reg;
 829         //reg=get_reg(regs[t+1].regmap_entry,r);
 830         //if(reg>=0) return reg;
 831       }
 832     }
 833   }
 834   return hr;
 835 }
 836
 837
 838 // Allocate every register, preserving source/target regs
 839 void alloc_all(struct regstat *cur,int i)
 840 {
 841   int hr;
 842
 843   for(hr=0;hr<HOST_REGS;hr++) {
 844     if(hr!=EXCLUDE_REG) {
 845       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 846          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 847       {
 848         cur->regmap[hr]=-1;
 849         cur->dirty&=~(1<<hr);
 850       }
 851       // Don't need zeros
 852       if((cur->regmap[hr]&63)==0)
 853       {
 854         cur->regmap[hr]=-1;
 855         cur->dirty&=~(1<<hr);
 856       }
 857     }
 858   }
 859 }
 860
 861 #ifndef NDEBUG
 862 static int host_tempreg_in_use;
 863
 864 static void host_tempreg_acquire(void)
 865 {
 866   assert(!host_tempreg_in_use);
 867   host_tempreg_in_use = 1;
 868 }
 869
 870 static void host_tempreg_release(void)
 871 {
 872   host_tempreg_in_use = 0;
 873 }
 874 #else
 875 static void host_tempreg_acquire(void) {}
 876 static void host_tempreg_release(void) {}
 877 #endif
 878
 879 #ifdef DRC_DBG
 880 extern void gen_interupt();
 881 extern void do_insn_cmp();
 882 #define FUNCNAME(f) { f, " " #f }
 883 static const struct {
 884   void *addr;
 885   const char *name;
 886 } function_names[] = {
 887   FUNCNAME(cc_interrupt),
 888   FUNCNAME(gen_interupt),
 889   FUNCNAME(get_addr_ht),
 890   FUNCNAME(get_addr),
 891   FUNCNAME(jump_handler_read8),
 892   FUNCNAME(jump_handler_read16),
 893   FUNCNAME(jump_handler_read32),
 894   FUNCNAME(jump_handler_write8),
 895   FUNCNAME(jump_handler_write16),
 896   FUNCNAME(jump_handler_write32),
 897   FUNCNAME(invalidate_addr),
 898   FUNCNAME(jump_to_new_pc),
 899   FUNCNAME(new_dyna_leave),
 900   FUNCNAME(pcsx_mtc0),
 901   FUNCNAME(pcsx_mtc0_ds),
 902   FUNCNAME(do_insn_cmp),
 903 #ifdef __arm__
 904   FUNCNAME(verify_code),
 905 #endif
 906 };
 907
 908 static const char *func_name(const void *a)
 909 {
 910   int i;
 911   for (i = 0; i < sizeof(function_names)/sizeof(function_names[0]); i++)
 912     if (function_names[i].addr == a)
 913       return function_names[i].name;
 914   return "";
 915 }
 916 #else
 917 #define func_name(x) ""
 918 #endif
 919
 920 #ifdef __i386__
 921 #include "assem_x86.c"
 922 #endif
 923 #ifdef __x86_64__
 924 #include "assem_x64.c"
 925 #endif
 926 #ifdef __arm__
 927 #include "assem_arm.c"
 928 #endif
 929 #ifdef __aarch64__
 930 #include "assem_arm64.c"
 931 #endif
 932
 933 static void *get_trampoline(const void *f)
 934 {
 935   size_t i;
 936
 937   for (i = 0; i < ARRAY_SIZE(ndrc->tramp.f); i++) {
 938     if (ndrc->tramp.f[i] == f || ndrc->tramp.f[i] == NULL)
 939       break;
 940   }
 941   if (i == ARRAY_SIZE(ndrc->tramp.f)) {
 942     SysPrintf("trampoline table is full, last func %p\n", f);
 943     abort();
 944   }
 945   if (ndrc->tramp.f[i] == NULL) {
 946     start_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
 947     ndrc->tramp.f[i] = f;
 948     end_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
 949   }
 950   return &ndrc->tramp.ops[i];
 951 }
 952
 953 static void emit_far_jump(const void *f)
 954 {
 955   if (can_jump_or_call(f)) {
 956     emit_jmp(f);
 957     return;
 958   }
 959
 960   f = get_trampoline(f);
 961   emit_jmp(f);
 962 }
 963
 964 static void emit_far_call(const void *f)
 965 {
 966   if (can_jump_or_call(f)) {
 967     emit_call(f);
 968     return;
 969   }
 970
 971   f = get_trampoline(f);
 972   emit_call(f);
 973 }
 974
 975 // Add virtual address mapping to linked list
 976 void ll_add(struct ll_entry **head,int vaddr,void *addr)
 977 {
 978   struct ll_entry *new_entry;
 979   new_entry=malloc(sizeof(struct ll_entry));
 980   assert(new_entry!=NULL);
 981   new_entry->vaddr=vaddr;
 982   new_entry->reg_sv_flags=0;
 983   new_entry->addr=addr;
 984   new_entry->next=*head;
 985   *head=new_entry;
 986 }
 987
 988 void ll_add_flags(struct ll_entry **head,int vaddr,u_int reg_sv_flags,void *addr)
 989 {
 990   ll_add(head,vaddr,addr);
 991   (*head)->reg_sv_flags=reg_sv_flags;
 992 }
 993
 994 // Check if an address is already compiled
 995 // but don't return addresses which are about to expire from the cache
 996 void *check_addr(u_int vaddr)
 997 {
 998   struct ht_entry *ht_bin = hash_table_get(vaddr);
 999   size_t i;
1000   for (i = 0; i < ARRAY_SIZE(ht_bin->vaddr); i++) {
1001     if (ht_bin->vaddr[i] == vaddr)
1002       if (doesnt_expire_soon((u_char *)ht_bin->tcaddr[i] - MAX_OUTPUT_BLOCK_SIZE))
1003         if (isclean(ht_bin->tcaddr[i]))
1004           return ht_bin->tcaddr[i];
1005   }
1006   u_int page=get_page(vaddr);
1007   struct ll_entry *head;
1008   head=jump_in[page];
1009   while (head != NULL) {
1010     if (head->vaddr == vaddr) {
1011       if (doesnt_expire_soon(head->addr)) {
1012         // Update existing entry with current address
1013         if (ht_bin->vaddr[0] == vaddr) {
1014           ht_bin->tcaddr[0] = head->addr;
1015           return head->addr;
1016         }
1017         if (ht_bin->vaddr[1] == vaddr) {
1018           ht_bin->tcaddr[1] = head->addr;
1019           return head->addr;
1020         }
1021         // Insert into hash table with low priority.
1022         // Don't evict existing entries, as they are probably
1023         // addresses that are being accessed frequently.
1024         if (ht_bin->vaddr[0] == -1) {
1025           ht_bin->vaddr[0] = vaddr;
1026           ht_bin->tcaddr[0] = head->addr;
1027         }
1028         else if (ht_bin->vaddr[1] == -1) {
1029           ht_bin->vaddr[1] = vaddr;
1030           ht_bin->tcaddr[1] = head->addr;
1031         }
1032         return head->addr;
1033       }
1034     }
1035     head=head->next;
1036   }
1037   return 0;
1038 }
1039
1040 void remove_hash(int vaddr)
1041 {
1042   //printf("remove hash: %x\n",vaddr);
1043   struct ht_entry *ht_bin = hash_table_get(vaddr);
1044   if (ht_bin->vaddr[1] == vaddr) {
1045     ht_bin->vaddr[1] = -1;
1046     ht_bin->tcaddr[1] = NULL;
1047   }
1048   if (ht_bin->vaddr[0] == vaddr) {
1049     ht_bin->vaddr[0] = ht_bin->vaddr[1];
1050     ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
1051     ht_bin->vaddr[1] = -1;
1052     ht_bin->tcaddr[1] = NULL;
1053   }
1054 }
1055
1056 void ll_remove_matching_addrs(struct ll_entry **head,uintptr_t addr,int shift)
1057 {
1058   struct ll_entry *next;
1059   while(*head) {
1060     if(((uintptr_t)((*head)->addr)>>shift)==(addr>>shift) ||
1061        ((uintptr_t)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
1062     {
1063       inv_debug("EXP: Remove pointer to %p (%x)\n",(*head)->addr,(*head)->vaddr);
1064       remove_hash((*head)->vaddr);
1065       next=(*head)->next;
1066       free(*head);
1067       *head=next;
1068     }
1069     else
1070     {
1071       head=&((*head)->next);
1072     }
1073   }
1074 }
1075
1076 // Remove all entries from linked list
1077 void ll_clear(struct ll_entry **head)
1078 {
1079   struct ll_entry *cur;
1080   struct ll_entry *next;
1081   if((cur=*head)) {
1082     *head=0;
1083     while(cur) {
1084       next=cur->next;
1085       free(cur);
1086       cur=next;
1087     }
1088   }
1089 }
1090
1091 // Dereference the pointers and remove if it matches
1092 static void ll_kill_pointers(struct ll_entry *head,uintptr_t addr,int shift)
1093 {
1094   while(head) {
1095     uintptr_t ptr = (uintptr_t)get_pointer(head->addr);
1096     inv_debug("EXP: Lookup pointer to %lx at %p (%x)\n",(long)ptr,head->addr,head->vaddr);
1097     if(((ptr>>shift)==(addr>>shift)) ||
1098        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
1099     {
1100       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
1101       void *host_addr=find_extjump_insn(head->addr);
1102       mark_clear_cache(host_addr);
1103       set_jump_target(host_addr, head->addr);
1104     }
1105     head=head->next;
1106   }
1107 }
1108
1109 // This is called when we write to a compiled block (see do_invstub)
1110 static void invalidate_page(u_int page)
1111 {
1112   struct ll_entry *head;
1113   struct ll_entry *next;
1114   head=jump_in[page];
1115   jump_in[page]=0;
1116   while(head!=NULL) {
1117     inv_debug("INVALIDATE: %x\n",head->vaddr);
1118     remove_hash(head->vaddr);
1119     next=head->next;
1120     free(head);
1121     head=next;
1122   }
1123   head=jump_out[page];
1124   jump_out[page]=0;
1125   while(head!=NULL) {
1126     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
1127     void *host_addr=find_extjump_insn(head->addr);
1128     mark_clear_cache(host_addr);
1129     set_jump_target(host_addr, head->addr);
1130     next=head->next;
1131     free(head);
1132     head=next;
1133   }
1134 }
1135
1136 static void invalidate_block_range(u_int block, u_int first, u_int last)
1137 {
1138   u_int page=get_page(block<<12);
1139   //printf("first=%d last=%d\n",first,last);
1140   invalidate_page(page);
1141   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1142   assert(last<page+5);
1143   // Invalidate the adjacent pages if a block crosses a 4K boundary
1144   while(first<page) {
1145     invalidate_page(first);
1146     first++;
1147   }
1148   for(first=page+1;first<last;first++) {
1149     invalidate_page(first);
1150   }
1151   do_clear_cache();
1152
1153   // Don't trap writes
1154   invalid_code[block]=1;
1155
1156   #ifdef USE_MINI_HT
1157   memset(mini_ht,-1,sizeof(mini_ht));
1158   #endif
1159 }
1160
1161 void invalidate_block(u_int block)
1162 {
1163   u_int page=get_page(block<<12);
1164   u_int vpage=get_vpage(block<<12);
1165   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1166   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1167   u_int first,last;
1168   first=last=page;
1169   struct ll_entry *head;
1170   head=jump_dirty[vpage];
1171   //printf("page=%d vpage=%d\n",page,vpage);
1172   while(head!=NULL) {
1173     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1174       u_char *start, *end;
1175       get_bounds(head->addr, &start, &end);
1176       //printf("start: %p end: %p\n", start, end);
1177       if (page < 2048 && start >= rdram && end < rdram+RAM_SIZE) {
1178         if (((start-rdram)>>12) <= page && ((end-1-rdram)>>12) >= page) {
1179           if ((((start-rdram)>>12)&2047) < first) first = ((start-rdram)>>12)&2047;
1180           if ((((end-1-rdram)>>12)&2047) > last)  last = ((end-1-rdram)>>12)&2047;
1181         }
1182       }
1183     }
1184     head=head->next;
1185   }
1186   invalidate_block_range(block,first,last);
1187 }
1188
1189 void invalidate_addr(u_int addr)
1190 {
1191   //static int rhits;
1192   // this check is done by the caller
1193   //if (inv_code_start<=addr&&addr<=inv_code_end) { rhits++; return; }
1194   u_int page=get_vpage(addr);
1195   if(page<2048) { // RAM
1196     struct ll_entry *head;
1197     u_int addr_min=~0, addr_max=0;
1198     u_int mask=RAM_SIZE-1;
1199     u_int addr_main=0x80000000|(addr&mask);
1200     int pg1;
1201     inv_code_start=addr_main&~0xfff;
1202     inv_code_end=addr_main|0xfff;
1203     pg1=page;
1204     if (pg1>0) {
1205       // must check previous page too because of spans..
1206       pg1--;
1207       inv_code_start-=0x1000;
1208     }
1209     for(;pg1<=page;pg1++) {
1210       for(head=jump_dirty[pg1];head!=NULL;head=head->next) {
1211         u_char *start_h, *end_h;
1212         u_int start, end;
1213         get_bounds(head->addr, &start_h, &end_h);
1214         start = (uintptr_t)start_h - ram_offset;
1215         end = (uintptr_t)end_h - ram_offset;
1216         if(start<=addr_main&&addr_main<end) {
1217           if(start<addr_min) addr_min=start;
1218           if(end>addr_max) addr_max=end;
1219         }
1220         else if(addr_main<start) {
1221           if(start<inv_code_end)
1222             inv_code_end=start-1;
1223         }
1224         else {
1225           if(end>inv_code_start)
1226             inv_code_start=end;
1227         }
1228       }
1229     }
1230     if (addr_min!=~0) {
1231       inv_debug("INV ADDR: %08x hit %08x-%08x\n", addr, addr_min, addr_max);
1232       inv_code_start=inv_code_end=~0;
1233       invalidate_block_range(addr>>12,(addr_min&mask)>>12,(addr_max&mask)>>12);
1234       return;
1235     }
1236     else {
1237       inv_code_start=(addr&~mask)|(inv_code_start&mask);
1238       inv_code_end=(addr&~mask)|(inv_code_end&mask);
1239       inv_debug("INV ADDR: %08x miss, inv %08x-%08x, sk %d\n", addr, inv_code_start, inv_code_end, 0);
1240       return;
1241     }
1242   }
1243   invalidate_block(addr>>12);
1244 }
1245
1246 // This is called when loading a save state.
1247 // Anything could have changed, so invalidate everything.
1248 void invalidate_all_pages(void)
1249 {
1250   u_int page;
1251   for(page=0;page<4096;page++)
1252     invalidate_page(page);
1253   for(page=0;page<1048576;page++)
1254     if(!invalid_code[page]) {
1255       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1256       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1257     }
1258   #ifdef USE_MINI_HT
1259   memset(mini_ht,-1,sizeof(mini_ht));
1260   #endif
1261   do_clear_cache();
1262 }
1263
1264 static void do_invstub(int n)
1265 {
1266   literal_pool(20);
1267   u_int reglist=stubs[n].a;
1268   set_jump_target(stubs[n].addr, out);
1269   save_regs(reglist);
1270   if(stubs[n].b!=0) emit_mov(stubs[n].b,0);
1271   emit_far_call(invalidate_addr);
1272   restore_regs(reglist);
1273   emit_jmp(stubs[n].retaddr); // return address
1274 }
1275
1276 // Add an entry to jump_out after making a link
1277 // src should point to code by emit_extjump2()
1278 void add_link(u_int vaddr,void *src)
1279 {
1280   u_int page=get_page(vaddr);
1281   inv_debug("add_link: %p -> %x (%d)\n",src,vaddr,page);
1282   check_extjump2(src);
1283   ll_add(jump_out+page,vaddr,src);
1284   //void *ptr=get_pointer(src);
1285   //inv_debug("add_link: Pointer is to %p\n",ptr);
1286 }
1287
1288 // If a code block was found to be unmodified (bit was set in
1289 // restore_candidate) and it remains unmodified (bit is clear
1290 // in invalid_code) then move the entries for that 4K page from
1291 // the dirty list to the clean list.
1292 void clean_blocks(u_int page)
1293 {
1294   struct ll_entry *head;
1295   inv_debug("INV: clean_blocks page=%d\n",page);
1296   head=jump_dirty[page];
1297   while(head!=NULL) {
1298     if(!invalid_code[head->vaddr>>12]) {
1299       // Don't restore blocks which are about to expire from the cache
1300       if (doesnt_expire_soon(head->addr)) {
1301         if(verify_dirty(head->addr)) {
1302           u_char *start, *end;
1303           //printf("Possibly Restore %x (%p)\n",head->vaddr, head->addr);
1304           u_int i;
1305           u_int inv=0;
1306           get_bounds(head->addr, &start, &end);
1307           if (start - rdram < RAM_SIZE) {
1308             for (i = (start-rdram+0x80000000)>>12; i <= (end-1-rdram+0x80000000)>>12; i++) {
1309               inv|=invalid_code[i];
1310             }
1311           }
1312           else if((signed int)head->vaddr>=(signed int)0x80000000+RAM_SIZE) {
1313             inv=1;
1314           }
1315           if(!inv) {
1316             void *clean_addr = get_clean_addr(head->addr);
1317             if (doesnt_expire_soon(clean_addr)) {
1318               u_int ppage=page;
1319               inv_debug("INV: Restored %x (%p/%p)\n",head->vaddr, head->addr, clean_addr);
1320               //printf("page=%x, addr=%x\n",page,head->vaddr);
1321               //assert(head->vaddr>>12==(page|0x80000));
1322               ll_add_flags(jump_in+ppage,head->vaddr,head->reg_sv_flags,clean_addr);
1323               struct ht_entry *ht_bin = hash_table_get(head->vaddr);
1324               if (ht_bin->vaddr[0] == head->vaddr)
1325                 ht_bin->tcaddr[0] = clean_addr; // Replace existing entry
1326               if (ht_bin->vaddr[1] == head->vaddr)
1327                 ht_bin->tcaddr[1] = clean_addr; // Replace existing entry
1328             }
1329           }
1330         }
1331       }
1332     }
1333     head=head->next;
1334   }
1335 }
1336
1337 /* Register allocation */
1338
1339 // Note: registers are allocated clean (unmodified state)
1340 // if you intend to modify the register, you must call dirty_reg().
1341 static void alloc_reg(struct regstat *cur,int i,signed char reg)
1342 {
1343   int r,hr;
1344   int preferred_reg = (reg&7);
1345   if(reg==CCREG) preferred_reg=HOST_CCREG;
1346   if(reg==PTEMP||reg==FTEMP) preferred_reg=12;
1347
1348   // Don't allocate unused registers
1349   if((cur->u>>reg)&1) return;
1350
1351   // see if it's already allocated
1352   for(hr=0;hr<HOST_REGS;hr++)
1353   {
1354     if(cur->regmap[hr]==reg) return;
1355   }
1356
1357   // Keep the same mapping if the register was already allocated in a loop
1358   preferred_reg = loop_reg(i,reg,preferred_reg);
1359
1360   // Try to allocate the preferred register
1361   if(cur->regmap[preferred_reg]==-1) {
1362     cur->regmap[preferred_reg]=reg;
1363     cur->dirty&=~(1<<preferred_reg);
1364     cur->isconst&=~(1<<preferred_reg);
1365     return;
1366   }
1367   r=cur->regmap[preferred_reg];
1368   assert(r < 64);
1369   if((cur->u>>r)&1) {
1370     cur->regmap[preferred_reg]=reg;
1371     cur->dirty&=~(1<<preferred_reg);
1372     cur->isconst&=~(1<<preferred_reg);
1373     return;
1374   }
1375
1376   // Clear any unneeded registers
1377   // We try to keep the mapping consistent, if possible, because it
1378   // makes branches easier (especially loops).  So we try to allocate
1379   // first (see above) before removing old mappings.  If this is not
1380   // possible then go ahead and clear out the registers that are no
1381   // longer needed.
1382   for(hr=0;hr<HOST_REGS;hr++)
1383   {
1384     r=cur->regmap[hr];
1385     if(r>=0) {
1386       assert(r < 64);
1387       if((cur->u>>r)&1) {cur->regmap[hr]=-1;break;}
1388     }
1389   }
1390   // Try to allocate any available register, but prefer
1391   // registers that have not been used recently.
1392   if(i>0) {
1393     for(hr=0;hr<HOST_REGS;hr++) {
1394       if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1395         if(regs[i-1].regmap[hr]!=rs1[i-1]&&regs[i-1].regmap[hr]!=rs2[i-1]&&regs[i-1].regmap[hr]!=rt1[i-1]&&regs[i-1].regmap[hr]!=rt2[i-1]) {
1396           cur->regmap[hr]=reg;
1397           cur->dirty&=~(1<<hr);
1398           cur->isconst&=~(1<<hr);
1399           return;
1400         }
1401       }
1402     }
1403   }
1404   // Try to allocate any available register
1405   for(hr=0;hr<HOST_REGS;hr++) {
1406     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1407       cur->regmap[hr]=reg;
1408       cur->dirty&=~(1<<hr);
1409       cur->isconst&=~(1<<hr);
1410       return;
1411     }
1412   }
1413
1414   // Ok, now we have to evict someone
1415   // Pick a register we hopefully won't need soon
1416   u_char hsn[MAXREG+1];
1417   memset(hsn,10,sizeof(hsn));
1418   int j;
1419   lsn(hsn,i,&preferred_reg);
1420   //printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",cur->regmap[0],cur->regmap[1],cur->regmap[2],cur->regmap[3],cur->regmap[5],cur->regmap[6],cur->regmap[7]);
1421   //printf("hsn(%x): %d %d %d %d %d %d %d\n",start+i*4,hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1422   if(i>0) {
1423     // Don't evict the cycle count at entry points, otherwise the entry
1424     // stub will have to write it.
1425     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1426     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1427     for(j=10;j>=3;j--)
1428     {
1429       // Alloc preferred register if available
1430       if(hsn[r=cur->regmap[preferred_reg]&63]==j) {
1431         for(hr=0;hr<HOST_REGS;hr++) {
1432           // Evict both parts of a 64-bit register
1433           if((cur->regmap[hr]&63)==r) {
1434             cur->regmap[hr]=-1;
1435             cur->dirty&=~(1<<hr);
1436             cur->isconst&=~(1<<hr);
1437           }
1438         }
1439         cur->regmap[preferred_reg]=reg;
1440         return;
1441       }
1442       for(r=1;r<=MAXREG;r++)
1443       {
1444         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1445           for(hr=0;hr<HOST_REGS;hr++) {
1446             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1447               if(cur->regmap[hr]==r) {
1448                 cur->regmap[hr]=reg;
1449                 cur->dirty&=~(1<<hr);
1450                 cur->isconst&=~(1<<hr);
1451                 return;
1452               }
1453             }
1454           }
1455         }
1456       }
1457     }
1458   }
1459   for(j=10;j>=0;j--)
1460   {
1461     for(r=1;r<=MAXREG;r++)
1462     {
1463       if(hsn[r]==j) {
1464         for(hr=0;hr<HOST_REGS;hr++) {
1465           if(cur->regmap[hr]==r) {
1466             cur->regmap[hr]=reg;
1467             cur->dirty&=~(1<<hr);
1468             cur->isconst&=~(1<<hr);
1469             return;
1470           }
1471         }
1472       }
1473     }
1474   }
1475   SysPrintf("This shouldn't happen (alloc_reg)");abort();
1476 }
1477
1478 // Allocate a temporary register.  This is done without regard to
1479 // dirty status or whether the register we request is on the unneeded list
1480 // Note: This will only allocate one register, even if called multiple times
1481 static void alloc_reg_temp(struct regstat *cur,int i,signed char reg)
1482 {
1483   int r,hr;
1484   int preferred_reg = -1;
1485
1486   // see if it's already allocated
1487   for(hr=0;hr<HOST_REGS;hr++)
1488   {
1489     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==reg) return;
1490   }
1491
1492   // Try to allocate any available register
1493   for(hr=HOST_REGS-1;hr>=0;hr--) {
1494     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1495       cur->regmap[hr]=reg;
1496       cur->dirty&=~(1<<hr);
1497       cur->isconst&=~(1<<hr);
1498       return;
1499     }
1500   }
1501
1502   // Find an unneeded register
1503   for(hr=HOST_REGS-1;hr>=0;hr--)
1504   {
1505     r=cur->regmap[hr];
1506     if(r>=0) {
1507       assert(r < 64);
1508       if((cur->u>>r)&1) {
1509         if(i==0||((unneeded_reg[i-1]>>r)&1)) {
1510           cur->regmap[hr]=reg;
1511           cur->dirty&=~(1<<hr);
1512           cur->isconst&=~(1<<hr);
1513           return;
1514         }
1515       }
1516     }
1517   }
1518
1519   // Ok, now we have to evict someone
1520   // Pick a register we hopefully won't need soon
1521   // TODO: we might want to follow unconditional jumps here
1522   // TODO: get rid of dupe code and make this into a function
1523   u_char hsn[MAXREG+1];
1524   memset(hsn,10,sizeof(hsn));
1525   int j;
1526   lsn(hsn,i,&preferred_reg);
1527   //printf("hsn: %d %d %d %d %d %d %d\n",hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1528   if(i>0) {
1529     // Don't evict the cycle count at entry points, otherwise the entry
1530     // stub will have to write it.
1531     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1532     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1533     for(j=10;j>=3;j--)
1534     {
1535       for(r=1;r<=MAXREG;r++)
1536       {
1537         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1538           for(hr=0;hr<HOST_REGS;hr++) {
1539             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1540               if(cur->regmap[hr]==r) {
1541                 cur->regmap[hr]=reg;
1542                 cur->dirty&=~(1<<hr);
1543                 cur->isconst&=~(1<<hr);
1544                 return;
1545               }
1546             }
1547           }
1548         }
1549       }
1550     }
1551   }
1552   for(j=10;j>=0;j--)
1553   {
1554     for(r=1;r<=MAXREG;r++)
1555     {
1556       if(hsn[r]==j) {
1557         for(hr=0;hr<HOST_REGS;hr++) {
1558           if(cur->regmap[hr]==r) {
1559             cur->regmap[hr]=reg;
1560             cur->dirty&=~(1<<hr);
1561             cur->isconst&=~(1<<hr);
1562             return;
1563           }
1564         }
1565       }
1566     }
1567   }
1568   SysPrintf("This shouldn't happen");abort();
1569 }
1570
1571 static void mov_alloc(struct regstat *current,int i)
1572 {
1573   // Note: Don't need to actually alloc the source registers
1574   //alloc_reg(current,i,rs1[i]);
1575   alloc_reg(current,i,rt1[i]);
1576
1577   clear_const(current,rs1[i]);
1578   clear_const(current,rt1[i]);
1579   dirty_reg(current,rt1[i]);
1580 }
1581
1582 static void shiftimm_alloc(struct regstat *current,int i)
1583 {
1584   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1585   {
1586     if(rt1[i]) {
1587       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1588       else lt1[i]=rs1[i];
1589       alloc_reg(current,i,rt1[i]);
1590       dirty_reg(current,rt1[i]);
1591       if(is_const(current,rs1[i])) {
1592         int v=get_const(current,rs1[i]);
1593         if(opcode2[i]==0x00) set_const(current,rt1[i],v<<imm[i]);
1594         if(opcode2[i]==0x02) set_const(current,rt1[i],(u_int)v>>imm[i]);
1595         if(opcode2[i]==0x03) set_const(current,rt1[i],v>>imm[i]);
1596       }
1597       else clear_const(current,rt1[i]);
1598     }
1599   }
1600   else
1601   {
1602     clear_const(current,rs1[i]);
1603     clear_const(current,rt1[i]);
1604   }
1605
1606   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1607   {
1608     assert(0);
1609   }
1610   if(opcode2[i]==0x3c) // DSLL32
1611   {
1612     assert(0);
1613   }
1614   if(opcode2[i]==0x3e) // DSRL32
1615   {
1616     assert(0);
1617   }
1618   if(opcode2[i]==0x3f) // DSRA32
1619   {
1620     assert(0);
1621   }
1622 }
1623
1624 static void shift_alloc(struct regstat *current,int i)
1625 {
1626   if(rt1[i]) {
1627     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1628     {
1629       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1630       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1631       alloc_reg(current,i,rt1[i]);
1632       if(rt1[i]==rs2[i]) {
1633         alloc_reg_temp(current,i,-1);
1634         minimum_free_regs[i]=1;
1635       }
1636     } else { // DSLLV/DSRLV/DSRAV
1637       assert(0);
1638     }
1639     clear_const(current,rs1[i]);
1640     clear_const(current,rs2[i]);
1641     clear_const(current,rt1[i]);
1642     dirty_reg(current,rt1[i]);
1643   }
1644 }
1645
1646 static void alu_alloc(struct regstat *current,int i)
1647 {
1648   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1649     if(rt1[i]) {
1650       if(rs1[i]&&rs2[i]) {
1651         alloc_reg(current,i,rs1[i]);
1652         alloc_reg(current,i,rs2[i]);
1653       }
1654       else {
1655         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1656         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1657       }
1658       alloc_reg(current,i,rt1[i]);
1659     }
1660   }
1661   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1662     if(rt1[i]) {
1663       alloc_reg(current,i,rs1[i]);
1664       alloc_reg(current,i,rs2[i]);
1665       alloc_reg(current,i,rt1[i]);
1666     }
1667   }
1668   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1669     if(rt1[i]) {
1670       if(rs1[i]&&rs2[i]) {
1671         alloc_reg(current,i,rs1[i]);
1672         alloc_reg(current,i,rs2[i]);
1673       }
1674       else
1675       {
1676         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1677         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1678       }
1679       alloc_reg(current,i,rt1[i]);
1680     }
1681   }
1682   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1683     assert(0);
1684   }
1685   clear_const(current,rs1[i]);
1686   clear_const(current,rs2[i]);
1687   clear_const(current,rt1[i]);
1688   dirty_reg(current,rt1[i]);
1689 }
1690
1691 static void imm16_alloc(struct regstat *current,int i)
1692 {
1693   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1694   else lt1[i]=rs1[i];
1695   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1696   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1697     assert(0);
1698   }
1699   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1700     clear_const(current,rs1[i]);
1701     clear_const(current,rt1[i]);
1702   }
1703   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1704     if(is_const(current,rs1[i])) {
1705       int v=get_const(current,rs1[i]);
1706       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1707       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1708       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1709     }
1710     else clear_const(current,rt1[i]);
1711   }
1712   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1713     if(is_const(current,rs1[i])) {
1714       int v=get_const(current,rs1[i]);
1715       set_const(current,rt1[i],v+imm[i]);
1716     }
1717     else clear_const(current,rt1[i]);
1718   }
1719   else {
1720     set_const(current,rt1[i],((long long)((short)imm[i]))<<16); // LUI
1721   }
1722   dirty_reg(current,rt1[i]);
1723 }
1724
1725 static void load_alloc(struct regstat *current,int i)
1726 {
1727   clear_const(current,rt1[i]);
1728   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1729   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1730   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1731   if(rt1[i]&&!((current->u>>rt1[i])&1)) {
1732     alloc_reg(current,i,rt1[i]);
1733     assert(get_reg(current->regmap,rt1[i])>=0);
1734     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1735     {
1736       assert(0);
1737     }
1738     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1739     {
1740       assert(0);
1741     }
1742     dirty_reg(current,rt1[i]);
1743     // LWL/LWR need a temporary register for the old value
1744     if(opcode[i]==0x22||opcode[i]==0x26)
1745     {
1746       alloc_reg(current,i,FTEMP);
1747       alloc_reg_temp(current,i,-1);
1748       minimum_free_regs[i]=1;
1749     }
1750   }
1751   else
1752   {
1753     // Load to r0 or unneeded register (dummy load)
1754     // but we still need a register to calculate the address
1755     if(opcode[i]==0x22||opcode[i]==0x26)
1756     {
1757       alloc_reg(current,i,FTEMP); // LWL/LWR need another temporary
1758     }
1759     alloc_reg_temp(current,i,-1);
1760     minimum_free_regs[i]=1;
1761     if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1762     {
1763       assert(0);
1764     }
1765   }
1766 }
1767
1768 void store_alloc(struct regstat *current,int i)
1769 {
1770   clear_const(current,rs2[i]);
1771   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1772   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1773   alloc_reg(current,i,rs2[i]);
1774   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1775     assert(0);
1776   }
1777   #if defined(HOST_IMM8)
1778   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1779   else alloc_reg(current,i,INVCP);
1780   #endif
1781   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) { // SWL/SWL/SDL/SDR
1782     alloc_reg(current,i,FTEMP);
1783   }
1784   // We need a temporary register for address generation
1785   alloc_reg_temp(current,i,-1);
1786   minimum_free_regs[i]=1;
1787 }
1788
1789 void c1ls_alloc(struct regstat *current,int i)
1790 {
1791   //clear_const(current,rs1[i]); // FIXME
1792   clear_const(current,rt1[i]);
1793   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1794   alloc_reg(current,i,CSREG); // Status
1795   alloc_reg(current,i,FTEMP);
1796   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1797     assert(0);
1798   }
1799   #if defined(HOST_IMM8)
1800   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1801   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1802     alloc_reg(current,i,INVCP);
1803   #endif
1804   // We need a temporary register for address generation
1805   alloc_reg_temp(current,i,-1);
1806 }
1807
1808 void c2ls_alloc(struct regstat *current,int i)
1809 {
1810   clear_const(current,rt1[i]);
1811   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1812   alloc_reg(current,i,FTEMP);
1813   #if defined(HOST_IMM8)
1814   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1815   if((opcode[i]&0x3b)==0x3a) // SWC2/SDC2
1816     alloc_reg(current,i,INVCP);
1817   #endif
1818   // We need a temporary register for address generation
1819   alloc_reg_temp(current,i,-1);
1820   minimum_free_regs[i]=1;
1821 }
1822
1823 #ifndef multdiv_alloc
1824 void multdiv_alloc(struct regstat *current,int i)
1825 {
1826   //  case 0x18: MULT
1827   //  case 0x19: MULTU
1828   //  case 0x1A: DIV
1829   //  case 0x1B: DIVU
1830   //  case 0x1C: DMULT
1831   //  case 0x1D: DMULTU
1832   //  case 0x1E: DDIV
1833   //  case 0x1F: DDIVU
1834   clear_const(current,rs1[i]);
1835   clear_const(current,rs2[i]);
1836   if(rs1[i]&&rs2[i])
1837   {
1838     if((opcode2[i]&4)==0) // 32-bit
1839     {
1840       current->u&=~(1LL<<HIREG);
1841       current->u&=~(1LL<<LOREG);
1842       alloc_reg(current,i,HIREG);
1843       alloc_reg(current,i,LOREG);
1844       alloc_reg(current,i,rs1[i]);
1845       alloc_reg(current,i,rs2[i]);
1846       dirty_reg(current,HIREG);
1847       dirty_reg(current,LOREG);
1848     }
1849     else // 64-bit
1850     {
1851       assert(0);
1852     }
1853   }
1854   else
1855   {
1856     // Multiply by zero is zero.
1857     // MIPS does not have a divide by zero exception.
1858     // The result is undefined, we return zero.
1859     alloc_reg(current,i,HIREG);
1860     alloc_reg(current,i,LOREG);
1861     dirty_reg(current,HIREG);
1862     dirty_reg(current,LOREG);
1863   }
1864 }
1865 #endif
1866
1867 void cop0_alloc(struct regstat *current,int i)
1868 {
1869   if(opcode2[i]==0) // MFC0
1870   {
1871     if(rt1[i]) {
1872       clear_const(current,rt1[i]);
1873       alloc_all(current,i);
1874       alloc_reg(current,i,rt1[i]);
1875       dirty_reg(current,rt1[i]);
1876     }
1877   }
1878   else if(opcode2[i]==4) // MTC0
1879   {
1880     if(rs1[i]){
1881       clear_const(current,rs1[i]);
1882       alloc_reg(current,i,rs1[i]);
1883       alloc_all(current,i);
1884     }
1885     else {
1886       alloc_all(current,i); // FIXME: Keep r0
1887       current->u&=~1LL;
1888       alloc_reg(current,i,0);
1889     }
1890   }
1891   else
1892   {
1893     // TLBR/TLBWI/TLBWR/TLBP/ERET
1894     assert(opcode2[i]==0x10);
1895     alloc_all(current,i);
1896   }
1897   minimum_free_regs[i]=HOST_REGS;
1898 }
1899
1900 static void cop12_alloc(struct regstat *current,int i)
1901 {
1902   alloc_reg(current,i,CSREG); // Load status
1903   if(opcode2[i]<3) // MFC1/CFC1
1904   {
1905     if(rt1[i]){
1906       clear_const(current,rt1[i]);
1907       alloc_reg(current,i,rt1[i]);
1908       dirty_reg(current,rt1[i]);
1909     }
1910     alloc_reg_temp(current,i,-1);
1911   }
1912   else if(opcode2[i]>3) // MTC1/CTC1
1913   {
1914     if(rs1[i]){
1915       clear_const(current,rs1[i]);
1916       alloc_reg(current,i,rs1[i]);
1917     }
1918     else {
1919       current->u&=~1LL;
1920       alloc_reg(current,i,0);
1921     }
1922     alloc_reg_temp(current,i,-1);
1923   }
1924   minimum_free_regs[i]=1;
1925 }
1926
1927 void c2op_alloc(struct regstat *current,int i)
1928 {
1929   alloc_reg_temp(current,i,-1);
1930 }
1931
1932 void syscall_alloc(struct regstat *current,int i)
1933 {
1934   alloc_cc(current,i);
1935   dirty_reg(current,CCREG);
1936   alloc_all(current,i);
1937   minimum_free_regs[i]=HOST_REGS;
1938   current->isconst=0;
1939 }
1940
1941 void delayslot_alloc(struct regstat *current,int i)
1942 {
1943   switch(itype[i]) {
1944     case UJUMP:
1945     case CJUMP:
1946     case SJUMP:
1947     case RJUMP:
1948     case SYSCALL:
1949     case HLECALL:
1950     case SPAN:
1951       assem_debug("jump in the delay slot.  this shouldn't happen.\n");//abort();
1952       SysPrintf("Disabled speculative precompilation\n");
1953       stop_after_jal=1;
1954       break;
1955     case IMM16:
1956       imm16_alloc(current,i);
1957       break;
1958     case LOAD:
1959     case LOADLR:
1960       load_alloc(current,i);
1961       break;
1962     case STORE:
1963     case STORELR:
1964       store_alloc(current,i);
1965       break;
1966     case ALU:
1967       alu_alloc(current,i);
1968       break;
1969     case SHIFT:
1970       shift_alloc(current,i);
1971       break;
1972     case MULTDIV:
1973       multdiv_alloc(current,i);
1974       break;
1975     case SHIFTIMM:
1976       shiftimm_alloc(current,i);
1977       break;
1978     case MOV:
1979       mov_alloc(current,i);
1980       break;
1981     case COP0:
1982       cop0_alloc(current,i);
1983       break;
1984     case COP1:
1985     case COP2:
1986       cop12_alloc(current,i);
1987       break;
1988     case C1LS:
1989       c1ls_alloc(current,i);
1990       break;
1991     case C2LS:
1992       c2ls_alloc(current,i);
1993       break;
1994     case C2OP:
1995       c2op_alloc(current,i);
1996       break;
1997   }
1998 }
1999
2000 // Special case where a branch and delay slot span two pages in virtual memory
2001 static void pagespan_alloc(struct regstat *current,int i)
2002 {
2003   current->isconst=0;
2004   current->wasconst=0;
2005   regs[i].wasconst=0;
2006   minimum_free_regs[i]=HOST_REGS;
2007   alloc_all(current,i);
2008   alloc_cc(current,i);
2009   dirty_reg(current,CCREG);
2010   if(opcode[i]==3) // JAL
2011   {
2012     alloc_reg(current,i,31);
2013     dirty_reg(current,31);
2014   }
2015   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
2016   {
2017     alloc_reg(current,i,rs1[i]);
2018     if (rt1[i]!=0) {
2019       alloc_reg(current,i,rt1[i]);
2020       dirty_reg(current,rt1[i]);
2021     }
2022   }
2023   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
2024   {
2025     if(rs1[i]) alloc_reg(current,i,rs1[i]);
2026     if(rs2[i]) alloc_reg(current,i,rs2[i]);
2027   }
2028   else
2029   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
2030   {
2031     if(rs1[i]) alloc_reg(current,i,rs1[i]);
2032   }
2033   //else ...
2034 }
2035
2036 static void add_stub(enum stub_type type, void *addr, void *retaddr,
2037   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e)
2038 {
2039   assert(stubcount < ARRAY_SIZE(stubs));
2040   stubs[stubcount].type = type;
2041   stubs[stubcount].addr = addr;
2042   stubs[stubcount].retaddr = retaddr;
2043   stubs[stubcount].a = a;
2044   stubs[stubcount].b = b;
2045   stubs[stubcount].c = c;
2046   stubs[stubcount].d = d;
2047   stubs[stubcount].e = e;
2048   stubcount++;
2049 }
2050
2051 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
2052   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist)
2053 {
2054   add_stub(type, addr, retaddr, i, addr_reg, (uintptr_t)i_regs, ccadj, reglist);
2055 }
2056
2057 // Write out a single register
2058 static void wb_register(signed char r,signed char regmap[],uint64_t dirty)
2059 {
2060   int hr;
2061   for(hr=0;hr<HOST_REGS;hr++) {
2062     if(hr!=EXCLUDE_REG) {
2063       if((regmap[hr]&63)==r) {
2064         if((dirty>>hr)&1) {
2065           assert(regmap[hr]<64);
2066           emit_storereg(r,hr);
2067         }
2068       }
2069     }
2070   }
2071 }
2072
2073 static void wb_valid(signed char pre[],signed char entry[],u_int dirty_pre,u_int dirty,uint64_t u)
2074 {
2075   //if(dirty_pre==dirty) return;
2076   int hr,reg;
2077   for(hr=0;hr<HOST_REGS;hr++) {
2078     if(hr!=EXCLUDE_REG) {
2079       reg=pre[hr];
2080       if(((~u)>>(reg&63))&1) {
2081         if(reg>0) {
2082           if(((dirty_pre&~dirty)>>hr)&1) {
2083             if(reg>0&&reg<34) {
2084               emit_storereg(reg,hr);
2085             }
2086             else if(reg>=64) {
2087               assert(0);
2088             }
2089           }
2090         }
2091       }
2092     }
2093   }
2094 }
2095
2096 // trashes r2
2097 static void pass_args(int a0, int a1)
2098 {
2099   if(a0==1&&a1==0) {
2100     // must swap
2101     emit_mov(a0,2); emit_mov(a1,1); emit_mov(2,0);
2102   }
2103   else if(a0!=0&&a1==0) {
2104     emit_mov(a1,1);
2105     if (a0>=0) emit_mov(a0,0);
2106   }
2107   else {
2108     if(a0>=0&&a0!=0) emit_mov(a0,0);
2109     if(a1>=0&&a1!=1) emit_mov(a1,1);
2110   }
2111 }
2112
2113 static void alu_assemble(int i,struct regstat *i_regs)
2114 {
2115   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2116     if(rt1[i]) {
2117       signed char s1,s2,t;
2118       t=get_reg(i_regs->regmap,rt1[i]);
2119       if(t>=0) {
2120         s1=get_reg(i_regs->regmap,rs1[i]);
2121         s2=get_reg(i_regs->regmap,rs2[i]);
2122         if(rs1[i]&&rs2[i]) {
2123           assert(s1>=0);
2124           assert(s2>=0);
2125           if(opcode2[i]&2) emit_sub(s1,s2,t);
2126           else emit_add(s1,s2,t);
2127         }
2128         else if(rs1[i]) {
2129           if(s1>=0) emit_mov(s1,t);
2130           else emit_loadreg(rs1[i],t);
2131         }
2132         else if(rs2[i]) {
2133           if(s2>=0) {
2134             if(opcode2[i]&2) emit_neg(s2,t);
2135             else emit_mov(s2,t);
2136           }
2137           else {
2138             emit_loadreg(rs2[i],t);
2139             if(opcode2[i]&2) emit_neg(t,t);
2140           }
2141         }
2142         else emit_zeroreg(t);
2143       }
2144     }
2145   }
2146   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2147     assert(0);
2148   }
2149   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2150     if(rt1[i]) {
2151       signed char s1l,s2l,t;
2152       {
2153         t=get_reg(i_regs->regmap,rt1[i]);
2154         //assert(t>=0);
2155         if(t>=0) {
2156           s1l=get_reg(i_regs->regmap,rs1[i]);
2157           s2l=get_reg(i_regs->regmap,rs2[i]);
2158           if(rs2[i]==0) // rx<r0
2159           {
2160             assert(s1l>=0);
2161             if(opcode2[i]==0x2a) // SLT
2162               emit_shrimm(s1l,31,t);
2163             else // SLTU (unsigned can not be less than zero)
2164               emit_zeroreg(t);
2165           }
2166           else if(rs1[i]==0) // r0<rx
2167           {
2168             assert(s2l>=0);
2169             if(opcode2[i]==0x2a) // SLT
2170               emit_set_gz32(s2l,t);
2171             else // SLTU (set if not zero)
2172               emit_set_nz32(s2l,t);
2173           }
2174           else{
2175             assert(s1l>=0);assert(s2l>=0);
2176             if(opcode2[i]==0x2a) // SLT
2177               emit_set_if_less32(s1l,s2l,t);
2178             else // SLTU
2179               emit_set_if_carry32(s1l,s2l,t);
2180           }
2181         }
2182       }
2183     }
2184   }
2185   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2186     if(rt1[i]) {
2187       signed char s1l,s2l,tl;
2188       tl=get_reg(i_regs->regmap,rt1[i]);
2189       {
2190         if(tl>=0) {
2191           s1l=get_reg(i_regs->regmap,rs1[i]);
2192           s2l=get_reg(i_regs->regmap,rs2[i]);
2193           if(rs1[i]&&rs2[i]) {
2194             assert(s1l>=0);
2195             assert(s2l>=0);
2196             if(opcode2[i]==0x24) { // AND
2197               emit_and(s1l,s2l,tl);
2198             } else
2199             if(opcode2[i]==0x25) { // OR
2200               emit_or(s1l,s2l,tl);
2201             } else
2202             if(opcode2[i]==0x26) { // XOR
2203               emit_xor(s1l,s2l,tl);
2204             } else
2205             if(opcode2[i]==0x27) { // NOR
2206               emit_or(s1l,s2l,tl);
2207               emit_not(tl,tl);
2208             }
2209           }
2210           else
2211           {
2212             if(opcode2[i]==0x24) { // AND
2213               emit_zeroreg(tl);
2214             } else
2215             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2216               if(rs1[i]){
2217                 if(s1l>=0) emit_mov(s1l,tl);
2218                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2219               }
2220               else
2221               if(rs2[i]){
2222                 if(s2l>=0) emit_mov(s2l,tl);
2223                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2224               }
2225               else emit_zeroreg(tl);
2226             } else
2227             if(opcode2[i]==0x27) { // NOR
2228               if(rs1[i]){
2229                 if(s1l>=0) emit_not(s1l,tl);
2230                 else {
2231                   emit_loadreg(rs1[i],tl);
2232                   emit_not(tl,tl);
2233                 }
2234               }
2235               else
2236               if(rs2[i]){
2237                 if(s2l>=0) emit_not(s2l,tl);
2238                 else {
2239                   emit_loadreg(rs2[i],tl);
2240                   emit_not(tl,tl);
2241                 }
2242               }
2243               else emit_movimm(-1,tl);
2244             }
2245           }
2246         }
2247       }
2248     }
2249   }
2250 }
2251
2252 void imm16_assemble(int i,struct regstat *i_regs)
2253 {
2254   if (opcode[i]==0x0f) { // LUI
2255     if(rt1[i]) {
2256       signed char t;
2257       t=get_reg(i_regs->regmap,rt1[i]);
2258       //assert(t>=0);
2259       if(t>=0) {
2260         if(!((i_regs->isconst>>t)&1))
2261           emit_movimm(imm[i]<<16,t);
2262       }
2263     }
2264   }
2265   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2266     if(rt1[i]) {
2267       signed char s,t;
2268       t=get_reg(i_regs->regmap,rt1[i]);
2269       s=get_reg(i_regs->regmap,rs1[i]);
2270       if(rs1[i]) {
2271         //assert(t>=0);
2272         //assert(s>=0);
2273         if(t>=0) {
2274           if(!((i_regs->isconst>>t)&1)) {
2275             if(s<0) {
2276               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2277               emit_addimm(t,imm[i],t);
2278             }else{
2279               if(!((i_regs->wasconst>>s)&1))
2280                 emit_addimm(s,imm[i],t);
2281               else
2282                 emit_movimm(constmap[i][s]+imm[i],t);
2283             }
2284           }
2285         }
2286       } else {
2287         if(t>=0) {
2288           if(!((i_regs->isconst>>t)&1))
2289             emit_movimm(imm[i],t);
2290         }
2291       }
2292     }
2293   }
2294   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2295     if(rt1[i]) {
2296       signed char sl,tl;
2297       tl=get_reg(i_regs->regmap,rt1[i]);
2298       sl=get_reg(i_regs->regmap,rs1[i]);
2299       if(tl>=0) {
2300         if(rs1[i]) {
2301           assert(sl>=0);
2302           emit_addimm(sl,imm[i],tl);
2303         } else {
2304           emit_movimm(imm[i],tl);
2305         }
2306       }
2307     }
2308   }
2309   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2310     if(rt1[i]) {
2311       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2312       signed char sl,t;
2313       t=get_reg(i_regs->regmap,rt1[i]);
2314       sl=get_reg(i_regs->regmap,rs1[i]);
2315       //assert(t>=0);
2316       if(t>=0) {
2317         if(rs1[i]>0) {
2318             if(opcode[i]==0x0a) { // SLTI
2319               if(sl<0) {
2320                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2321                 emit_slti32(t,imm[i],t);
2322               }else{
2323                 emit_slti32(sl,imm[i],t);
2324               }
2325             }
2326             else { // SLTIU
2327               if(sl<0) {
2328                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2329                 emit_sltiu32(t,imm[i],t);
2330               }else{
2331                 emit_sltiu32(sl,imm[i],t);
2332               }
2333             }
2334         }else{
2335           // SLTI(U) with r0 is just stupid,
2336           // nonetheless examples can be found
2337           if(opcode[i]==0x0a) // SLTI
2338             if(0<imm[i]) emit_movimm(1,t);
2339             else emit_zeroreg(t);
2340           else // SLTIU
2341           {
2342             if(imm[i]) emit_movimm(1,t);
2343             else emit_zeroreg(t);
2344           }
2345         }
2346       }
2347     }
2348   }
2349   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2350     if(rt1[i]) {
2351       signed char sl,tl;
2352       tl=get_reg(i_regs->regmap,rt1[i]);
2353       sl=get_reg(i_regs->regmap,rs1[i]);
2354       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2355         if(opcode[i]==0x0c) //ANDI
2356         {
2357           if(rs1[i]) {
2358             if(sl<0) {
2359               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2360               emit_andimm(tl,imm[i],tl);
2361             }else{
2362               if(!((i_regs->wasconst>>sl)&1))
2363                 emit_andimm(sl,imm[i],tl);
2364               else
2365                 emit_movimm(constmap[i][sl]&imm[i],tl);
2366             }
2367           }
2368           else
2369             emit_zeroreg(tl);
2370         }
2371         else
2372         {
2373           if(rs1[i]) {
2374             if(sl<0) {
2375               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2376             }
2377             if(opcode[i]==0x0d) { // ORI
2378               if(sl<0) {
2379                 emit_orimm(tl,imm[i],tl);
2380               }else{
2381                 if(!((i_regs->wasconst>>sl)&1))
2382                   emit_orimm(sl,imm[i],tl);
2383                 else
2384                   emit_movimm(constmap[i][sl]|imm[i],tl);
2385               }
2386             }
2387             if(opcode[i]==0x0e) { // XORI
2388               if(sl<0) {
2389                 emit_xorimm(tl,imm[i],tl);
2390               }else{
2391                 if(!((i_regs->wasconst>>sl)&1))
2392                   emit_xorimm(sl,imm[i],tl);
2393                 else
2394                   emit_movimm(constmap[i][sl]^imm[i],tl);
2395               }
2396             }
2397           }
2398           else {
2399             emit_movimm(imm[i],tl);
2400           }
2401         }
2402       }
2403     }
2404   }
2405 }
2406
2407 void shiftimm_assemble(int i,struct regstat *i_regs)
2408 {
2409   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2410   {
2411     if(rt1[i]) {
2412       signed char s,t;
2413       t=get_reg(i_regs->regmap,rt1[i]);
2414       s=get_reg(i_regs->regmap,rs1[i]);
2415       //assert(t>=0);
2416       if(t>=0&&!((i_regs->isconst>>t)&1)){
2417         if(rs1[i]==0)
2418         {
2419           emit_zeroreg(t);
2420         }
2421         else
2422         {
2423           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2424           if(imm[i]) {
2425             if(opcode2[i]==0) // SLL
2426             {
2427               emit_shlimm(s<0?t:s,imm[i],t);
2428             }
2429             if(opcode2[i]==2) // SRL
2430             {
2431               emit_shrimm(s<0?t:s,imm[i],t);
2432             }
2433             if(opcode2[i]==3) // SRA
2434             {
2435               emit_sarimm(s<0?t:s,imm[i],t);
2436             }
2437           }else{
2438             // Shift by zero
2439             if(s>=0 && s!=t) emit_mov(s,t);
2440           }
2441         }
2442       }
2443       //emit_storereg(rt1[i],t); //DEBUG
2444     }
2445   }
2446   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2447   {
2448     assert(0);
2449   }
2450   if(opcode2[i]==0x3c) // DSLL32
2451   {
2452     assert(0);
2453   }
2454   if(opcode2[i]==0x3e) // DSRL32
2455   {
2456     assert(0);
2457   }
2458   if(opcode2[i]==0x3f) // DSRA32
2459   {
2460     assert(0);
2461   }
2462 }
2463
2464 #ifndef shift_assemble
2465 static void shift_assemble(int i,struct regstat *i_regs)
2466 {
2467   signed char s,t,shift;
2468   if (rt1[i] == 0)
2469     return;
2470   assert(opcode2[i]<=0x07); // SLLV/SRLV/SRAV
2471   t = get_reg(i_regs->regmap, rt1[i]);
2472   s = get_reg(i_regs->regmap, rs1[i]);
2473   shift = get_reg(i_regs->regmap, rs2[i]);
2474   if (t < 0)
2475     return;
2476
2477   if(rs1[i]==0)
2478     emit_zeroreg(t);
2479   else if(rs2[i]==0) {
2480     assert(s>=0);
2481     if(s!=t) emit_mov(s,t);
2482   }
2483   else {
2484     host_tempreg_acquire();
2485     emit_andimm(shift,31,HOST_TEMPREG);
2486     switch(opcode2[i]) {
2487     case 4: // SLLV
2488       emit_shl(s,HOST_TEMPREG,t);
2489       break;
2490     case 6: // SRLV
2491       emit_shr(s,HOST_TEMPREG,t);
2492       break;
2493     case 7: // SRAV
2494       emit_sar(s,HOST_TEMPREG,t);
2495       break;
2496     default:
2497       assert(0);
2498     }
2499     host_tempreg_release();
2500   }
2501 }
2502
2503 #endif
2504
2505 enum {
2506   MTYPE_8000 = 0,
2507   MTYPE_8020,
2508   MTYPE_0000,
2509   MTYPE_A000,
2510   MTYPE_1F80,
2511 };
2512
2513 static int get_ptr_mem_type(u_int a)
2514 {
2515   if(a < 0x00200000) {
2516     if(a<0x1000&&((start>>20)==0xbfc||(start>>24)==0xa0))
2517       // return wrong, must use memhandler for BIOS self-test to pass
2518       // 007 does similar stuff from a00 mirror, weird stuff
2519       return MTYPE_8000;
2520     return MTYPE_0000;
2521   }
2522   if(0x1f800000 <= a && a < 0x1f801000)
2523     return MTYPE_1F80;
2524   if(0x80200000 <= a && a < 0x80800000)
2525     return MTYPE_8020;
2526   if(0xa0000000 <= a && a < 0xa0200000)
2527     return MTYPE_A000;
2528   return MTYPE_8000;
2529 }
2530
2531 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override)
2532 {
2533   void *jaddr = NULL;
2534   int type=0;
2535   int mr=rs1[i];
2536   if(((smrv_strong|smrv_weak)>>mr)&1) {
2537     type=get_ptr_mem_type(smrv[mr]);
2538     //printf("set %08x @%08x r%d %d\n", smrv[mr], start+i*4, mr, type);
2539   }
2540   else {
2541     // use the mirror we are running on
2542     type=get_ptr_mem_type(start);
2543     //printf("set nospec   @%08x r%d %d\n", start+i*4, mr, type);
2544   }
2545
2546   if(type==MTYPE_8020) { // RAM 80200000+ mirror
2547     host_tempreg_acquire();
2548     emit_andimm(addr,~0x00e00000,HOST_TEMPREG);
2549     addr=*addr_reg_override=HOST_TEMPREG;
2550     type=0;
2551   }
2552   else if(type==MTYPE_0000) { // RAM 0 mirror
2553     host_tempreg_acquire();
2554     emit_orimm(addr,0x80000000,HOST_TEMPREG);
2555     addr=*addr_reg_override=HOST_TEMPREG;
2556     type=0;
2557   }
2558   else if(type==MTYPE_A000) { // RAM A mirror
2559     host_tempreg_acquire();
2560     emit_andimm(addr,~0x20000000,HOST_TEMPREG);
2561     addr=*addr_reg_override=HOST_TEMPREG;
2562     type=0;
2563   }
2564   else if(type==MTYPE_1F80) { // scratchpad
2565     if (psxH == (void *)0x1f800000) {
2566       host_tempreg_acquire();
2567       emit_xorimm(addr,0x1f800000,HOST_TEMPREG);
2568       emit_cmpimm(HOST_TEMPREG,0x1000);
2569       host_tempreg_release();
2570       jaddr=out;
2571       emit_jc(0);
2572     }
2573     else {
2574       // do the usual RAM check, jump will go to the right handler
2575       type=0;
2576     }
2577   }
2578
2579   if(type==0)
2580   {
2581     emit_cmpimm(addr,RAM_SIZE);
2582     jaddr=out;
2583     #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2584     // Hint to branch predictor that the branch is unlikely to be taken
2585     if(rs1[i]>=28)
2586       emit_jno_unlikely(0);
2587     else
2588     #endif
2589       emit_jno(0);
2590     if(ram_offset!=0) {
2591       host_tempreg_acquire();
2592       emit_addimm(addr,ram_offset,HOST_TEMPREG);
2593       addr=*addr_reg_override=HOST_TEMPREG;
2594     }
2595   }
2596
2597   return jaddr;
2598 }
2599
2600 // return memhandler, or get directly accessable address and return 0
2601 static void *get_direct_memhandler(void *table, u_int addr,
2602   enum stub_type type, uintptr_t *addr_host)
2603 {
2604   uintptr_t l1, l2 = 0;
2605   l1 = ((uintptr_t *)table)[addr>>12];
2606   if ((l1 & (1ul << (sizeof(l1)*8-1))) == 0) {
2607     uintptr_t v = l1 << 1;
2608     *addr_host = v + addr;
2609     return NULL;
2610   }
2611   else {
2612     l1 <<= 1;
2613     if (type == LOADB_STUB || type == LOADBU_STUB || type == STOREB_STUB)
2614       l2 = ((uintptr_t *)l1)[0x1000/4 + 0x1000/2 + (addr&0xfff)];
2615     else if (type == LOADH_STUB || type == LOADHU_STUB || type == STOREH_STUB)
2616       l2=((uintptr_t *)l1)[0x1000/4 + (addr&0xfff)/2];
2617     else
2618       l2=((uintptr_t *)l1)[(addr&0xfff)/4];
2619     if ((l2 & (1<<31)) == 0) {
2620       uintptr_t v = l2 << 1;
2621       *addr_host = v + (addr&0xfff);
2622       return NULL;
2623     }
2624     return (void *)(l2 << 1);
2625   }
2626 }
2627
2628 static void load_assemble(int i,struct regstat *i_regs)
2629 {
2630   int s,tl,addr;
2631   int offset;
2632   void *jaddr=0;
2633   int memtarget=0,c=0;
2634   int fastio_reg_override=-1;
2635   u_int hr,reglist=0;
2636   tl=get_reg(i_regs->regmap,rt1[i]);
2637   s=get_reg(i_regs->regmap,rs1[i]);
2638   offset=imm[i];
2639   for(hr=0;hr<HOST_REGS;hr++) {
2640     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2641   }
2642   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2643   if(s>=0) {
2644     c=(i_regs->wasconst>>s)&1;
2645     if (c) {
2646       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2647     }
2648   }
2649   //printf("load_assemble: c=%d\n",c);
2650   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2651   // FIXME: Even if the load is a NOP, we should check for pagefaults...
2652   if((tl<0&&(!c||(((u_int)constmap[i][s]+offset)>>16)==0x1f80))
2653     ||rt1[i]==0) {
2654       // could be FIFO, must perform the read
2655       // ||dummy read
2656       assem_debug("(forced read)\n");
2657       tl=get_reg(i_regs->regmap,-1);
2658       assert(tl>=0);
2659   }
2660   if(offset||s<0||c) addr=tl;
2661   else addr=s;
2662   //if(tl<0) tl=get_reg(i_regs->regmap,-1);
2663  if(tl>=0) {
2664   //printf("load_assemble: c=%d\n",c);
2665   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2666   assert(tl>=0); // Even if the load is a NOP, we must check for pagefaults and I/O
2667   reglist&=~(1<<tl);
2668   if(!c) {
2669     #ifdef R29_HACK
2670     // Strmnnrmn's speed hack
2671     if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
2672     #endif
2673     {
2674       jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override);
2675     }
2676   }
2677   else if(ram_offset&&memtarget) {
2678     host_tempreg_acquire();
2679     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2680     fastio_reg_override=HOST_TEMPREG;
2681   }
2682   int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg
2683   if (opcode[i]==0x20) { // LB
2684     if(!c||memtarget) {
2685       if(!dummy) {
2686         {
2687           int x=0,a=tl;
2688           if(!c) a=addr;
2689           if(fastio_reg_override>=0) a=fastio_reg_override;
2690
2691           emit_movsbl_indexed(x,a,tl);
2692         }
2693       }
2694       if(jaddr)
2695         add_stub_r(LOADB_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2696     }
2697     else
2698       inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2699   }
2700   if (opcode[i]==0x21) { // LH
2701     if(!c||memtarget) {
2702       if(!dummy) {
2703         int x=0,a=tl;
2704         if(!c) a=addr;
2705         if(fastio_reg_override>=0) a=fastio_reg_override;
2706         emit_movswl_indexed(x,a,tl);
2707       }
2708       if(jaddr)
2709         add_stub_r(LOADH_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2710     }
2711     else
2712       inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2713   }
2714   if (opcode[i]==0x23) { // LW
2715     if(!c||memtarget) {
2716       if(!dummy) {
2717         int a=addr;
2718         if(fastio_reg_override>=0) a=fastio_reg_override;
2719         emit_readword_indexed(0,a,tl);
2720       }
2721       if(jaddr)
2722         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2723     }
2724     else
2725       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2726   }
2727   if (opcode[i]==0x24) { // LBU
2728     if(!c||memtarget) {
2729       if(!dummy) {
2730         int x=0,a=tl;
2731         if(!c) a=addr;
2732         if(fastio_reg_override>=0) a=fastio_reg_override;
2733
2734         emit_movzbl_indexed(x,a,tl);
2735       }
2736       if(jaddr)
2737         add_stub_r(LOADBU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2738     }
2739     else
2740       inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2741   }
2742   if (opcode[i]==0x25) { // LHU
2743     if(!c||memtarget) {
2744       if(!dummy) {
2745         int x=0,a=tl;
2746         if(!c) a=addr;
2747         if(fastio_reg_override>=0) a=fastio_reg_override;
2748         emit_movzwl_indexed(x,a,tl);
2749       }
2750       if(jaddr)
2751         add_stub_r(LOADHU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2752     }
2753     else
2754       inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2755   }
2756   if (opcode[i]==0x27) { // LWU
2757     assert(0);
2758   }
2759   if (opcode[i]==0x37) { // LD
2760     assert(0);
2761   }
2762  }
2763  if (fastio_reg_override == HOST_TEMPREG)
2764    host_tempreg_release();
2765 }
2766
2767 #ifndef loadlr_assemble
2768 static void loadlr_assemble(int i,struct regstat *i_regs)
2769 {
2770   int s,tl,temp,temp2,addr;
2771   int offset;
2772   void *jaddr=0;
2773   int memtarget=0,c=0;
2774   int fastio_reg_override=-1;
2775   u_int hr,reglist=0;
2776   tl=get_reg(i_regs->regmap,rt1[i]);
2777   s=get_reg(i_regs->regmap,rs1[i]);
2778   temp=get_reg(i_regs->regmap,-1);
2779   temp2=get_reg(i_regs->regmap,FTEMP);
2780   addr=get_reg(i_regs->regmap,AGEN1+(i&1));
2781   assert(addr<0);
2782   offset=imm[i];
2783   for(hr=0;hr<HOST_REGS;hr++) {
2784     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2785   }
2786   reglist|=1<<temp;
2787   if(offset||s<0||c) addr=temp2;
2788   else addr=s;
2789   if(s>=0) {
2790     c=(i_regs->wasconst>>s)&1;
2791     if(c) {
2792       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2793     }
2794   }
2795   if(!c) {
2796     emit_shlimm(addr,3,temp);
2797     if (opcode[i]==0x22||opcode[i]==0x26) {
2798       emit_andimm(addr,0xFFFFFFFC,temp2); // LWL/LWR
2799     }else{
2800       emit_andimm(addr,0xFFFFFFF8,temp2); // LDL/LDR
2801     }
2802     jaddr=emit_fastpath_cmp_jump(i,temp2,&fastio_reg_override);
2803   }
2804   else {
2805     if(ram_offset&&memtarget) {
2806       host_tempreg_acquire();
2807       emit_addimm(temp2,ram_offset,HOST_TEMPREG);
2808       fastio_reg_override=HOST_TEMPREG;
2809     }
2810     if (opcode[i]==0x22||opcode[i]==0x26) {
2811       emit_movimm(((constmap[i][s]+offset)<<3)&24,temp); // LWL/LWR
2812     }else{
2813       emit_movimm(((constmap[i][s]+offset)<<3)&56,temp); // LDL/LDR
2814     }
2815   }
2816   if (opcode[i]==0x22||opcode[i]==0x26) { // LWL/LWR
2817     if(!c||memtarget) {
2818       int a=temp2;
2819       if(fastio_reg_override>=0) a=fastio_reg_override;
2820       emit_readword_indexed(0,a,temp2);
2821       if(fastio_reg_override==HOST_TEMPREG) host_tempreg_release();
2822       if(jaddr) add_stub_r(LOADW_STUB,jaddr,out,i,temp2,i_regs,ccadj[i],reglist);
2823     }
2824     else
2825       inline_readstub(LOADW_STUB,i,(constmap[i][s]+offset)&0xFFFFFFFC,i_regs->regmap,FTEMP,ccadj[i],reglist);
2826     if(rt1[i]) {
2827       assert(tl>=0);
2828       emit_andimm(temp,24,temp);
2829       if (opcode[i]==0x22) // LWL
2830         emit_xorimm(temp,24,temp);
2831       host_tempreg_acquire();
2832       emit_movimm(-1,HOST_TEMPREG);
2833       if (opcode[i]==0x26) {
2834         emit_shr(temp2,temp,temp2);
2835         emit_bic_lsr(tl,HOST_TEMPREG,temp,tl);
2836       }else{
2837         emit_shl(temp2,temp,temp2);
2838         emit_bic_lsl(tl,HOST_TEMPREG,temp,tl);
2839       }
2840       host_tempreg_release();
2841       emit_or(temp2,tl,tl);
2842     }
2843     //emit_storereg(rt1[i],tl); // DEBUG
2844   }
2845   if (opcode[i]==0x1A||opcode[i]==0x1B) { // LDL/LDR
2846     assert(0);
2847   }
2848 }
2849 #endif
2850
2851 void store_assemble(int i,struct regstat *i_regs)
2852 {
2853   int s,tl;
2854   int addr,temp;
2855   int offset;
2856   void *jaddr=0;
2857   enum stub_type type;
2858   int memtarget=0,c=0;
2859   int agr=AGEN1+(i&1);
2860   int fastio_reg_override=-1;
2861   u_int hr,reglist=0;
2862   tl=get_reg(i_regs->regmap,rs2[i]);
2863   s=get_reg(i_regs->regmap,rs1[i]);
2864   temp=get_reg(i_regs->regmap,agr);
2865   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2866   offset=imm[i];
2867   if(s>=0) {
2868     c=(i_regs->wasconst>>s)&1;
2869     if(c) {
2870       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2871     }
2872   }
2873   assert(tl>=0);
2874   assert(temp>=0);
2875   for(hr=0;hr<HOST_REGS;hr++) {
2876     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2877   }
2878   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2879   if(offset||s<0||c) addr=temp;
2880   else addr=s;
2881   if(!c) {
2882     jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override);
2883   }
2884   else if(ram_offset&&memtarget) {
2885     host_tempreg_acquire();
2886     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2887     fastio_reg_override=HOST_TEMPREG;
2888   }
2889
2890   if (opcode[i]==0x28) { // SB
2891     if(!c||memtarget) {
2892       int x=0,a=temp;
2893       if(!c) a=addr;
2894       if(fastio_reg_override>=0) a=fastio_reg_override;
2895       emit_writebyte_indexed(tl,x,a);
2896     }
2897     type=STOREB_STUB;
2898   }
2899   if (opcode[i]==0x29) { // SH
2900     if(!c||memtarget) {
2901       int x=0,a=temp;
2902       if(!c) a=addr;
2903       if(fastio_reg_override>=0) a=fastio_reg_override;
2904       emit_writehword_indexed(tl,x,a);
2905     }
2906     type=STOREH_STUB;
2907   }
2908   if (opcode[i]==0x2B) { // SW
2909     if(!c||memtarget) {
2910       int a=addr;
2911       if(fastio_reg_override>=0) a=fastio_reg_override;
2912       emit_writeword_indexed(tl,0,a);
2913     }
2914     type=STOREW_STUB;
2915   }
2916   if (opcode[i]==0x3F) { // SD
2917     assert(0);
2918     type=STORED_STUB;
2919   }
2920   if(fastio_reg_override==HOST_TEMPREG)
2921     host_tempreg_release();
2922   if(jaddr) {
2923     // PCSX store handlers don't check invcode again
2924     reglist|=1<<addr;
2925     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2926     jaddr=0;
2927   }
2928   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
2929     if(!c||memtarget) {
2930       #ifdef DESTRUCTIVE_SHIFT
2931       // The x86 shift operation is 'destructive'; it overwrites the
2932       // source register, so we need to make a copy first and use that.
2933       addr=temp;
2934       #endif
2935       #if defined(HOST_IMM8)
2936       int ir=get_reg(i_regs->regmap,INVCP);
2937       assert(ir>=0);
2938       emit_cmpmem_indexedsr12_reg(ir,addr,1);
2939       #else
2940       emit_cmpmem_indexedsr12_imm(invalid_code,addr,1);
2941       #endif
2942       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2943       emit_callne(invalidate_addr_reg[addr]);
2944       #else
2945       void *jaddr2 = out;
2946       emit_jne(0);
2947       add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),addr,0,0,0);
2948       #endif
2949     }
2950   }
2951   u_int addr_val=constmap[i][s]+offset;
2952   if(jaddr) {
2953     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2954   } else if(c&&!memtarget) {
2955     inline_writestub(type,i,addr_val,i_regs->regmap,rs2[i],ccadj[i],reglist);
2956   }
2957   // basic current block modification detection..
2958   // not looking back as that should be in mips cache already
2959   // (see Spyro2 title->attract mode)
2960   if(c&&start+i*4<addr_val&&addr_val<start+slen*4) {
2961     SysPrintf("write to %08x hits block %08x, pc=%08x\n",addr_val,start,start+i*4);
2962     assert(i_regs->regmap==regs[i].regmap); // not delay slot
2963     if(i_regs->regmap==regs[i].regmap) {
2964       load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
2965       wb_dirtys(regs[i].regmap_entry,regs[i].wasdirty);
2966       emit_movimm(start+i*4+4,0);
2967       emit_writeword(0,&pcaddr);
2968       emit_addimm(HOST_CCREG,2,HOST_CCREG);
2969       emit_far_call(get_addr_ht);
2970       emit_jmpreg(0);
2971     }
2972   }
2973 }
2974
2975 static void storelr_assemble(int i,struct regstat *i_regs)
2976 {
2977   int s,tl;
2978   int temp;
2979   int offset;
2980   void *jaddr=0;
2981   void *case1, *case2, *case3;
2982   void *done0, *done1, *done2;
2983   int memtarget=0,c=0;
2984   int agr=AGEN1+(i&1);
2985   u_int hr,reglist=0;
2986   tl=get_reg(i_regs->regmap,rs2[i]);
2987   s=get_reg(i_regs->regmap,rs1[i]);
2988   temp=get_reg(i_regs->regmap,agr);
2989   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2990   offset=imm[i];
2991   if(s>=0) {
2992     c=(i_regs->isconst>>s)&1;
2993     if(c) {
2994       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2995     }
2996   }
2997   assert(tl>=0);
2998   for(hr=0;hr<HOST_REGS;hr++) {
2999     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3000   }
3001   assert(temp>=0);
3002   if(!c) {
3003     emit_cmpimm(s<0||offset?temp:s,RAM_SIZE);
3004     if(!offset&&s!=temp) emit_mov(s,temp);
3005     jaddr=out;
3006     emit_jno(0);
3007   }
3008   else
3009   {
3010     if(!memtarget||!rs1[i]) {
3011       jaddr=out;
3012       emit_jmp(0);
3013     }
3014   }
3015   if(ram_offset)
3016     emit_addimm_no_flags(ram_offset,temp);
3017
3018   if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
3019     assert(0);
3020   }
3021
3022   emit_xorimm(temp,3,temp);
3023   emit_testimm(temp,2);
3024   case2=out;
3025   emit_jne(0);
3026   emit_testimm(temp,1);
3027   case1=out;
3028   emit_jne(0);
3029   // 0
3030   if (opcode[i]==0x2A) { // SWL
3031     emit_writeword_indexed(tl,0,temp);
3032   }
3033   else if (opcode[i]==0x2E) { // SWR
3034     emit_writebyte_indexed(tl,3,temp);
3035   }
3036   else
3037     assert(0);
3038   done0=out;
3039   emit_jmp(0);
3040   // 1
3041   set_jump_target(case1, out);
3042   if (opcode[i]==0x2A) { // SWL
3043     // Write 3 msb into three least significant bytes
3044     if(rs2[i]) emit_rorimm(tl,8,tl);
3045     emit_writehword_indexed(tl,-1,temp);
3046     if(rs2[i]) emit_rorimm(tl,16,tl);
3047     emit_writebyte_indexed(tl,1,temp);
3048     if(rs2[i]) emit_rorimm(tl,8,tl);
3049   }
3050   else if (opcode[i]==0x2E) { // SWR
3051     // Write two lsb into two most significant bytes
3052     emit_writehword_indexed(tl,1,temp);
3053   }
3054   done1=out;
3055   emit_jmp(0);
3056   // 2
3057   set_jump_target(case2, out);
3058   emit_testimm(temp,1);
3059   case3=out;
3060   emit_jne(0);
3061   if (opcode[i]==0x2A) { // SWL
3062     // Write two msb into two least significant bytes
3063     if(rs2[i]) emit_rorimm(tl,16,tl);
3064     emit_writehword_indexed(tl,-2,temp);
3065     if(rs2[i]) emit_rorimm(tl,16,tl);
3066   }
3067   else if (opcode[i]==0x2E) { // SWR
3068     // Write 3 lsb into three most significant bytes
3069     emit_writebyte_indexed(tl,-1,temp);
3070     if(rs2[i]) emit_rorimm(tl,8,tl);
3071     emit_writehword_indexed(tl,0,temp);
3072     if(rs2[i]) emit_rorimm(tl,24,tl);
3073   }
3074   done2=out;
3075   emit_jmp(0);
3076   // 3
3077   set_jump_target(case3, out);
3078   if (opcode[i]==0x2A) { // SWL
3079     // Write msb into least significant byte
3080     if(rs2[i]) emit_rorimm(tl,24,tl);
3081     emit_writebyte_indexed(tl,-3,temp);
3082     if(rs2[i]) emit_rorimm(tl,8,tl);
3083   }
3084   else if (opcode[i]==0x2E) { // SWR
3085     // Write entire word
3086     emit_writeword_indexed(tl,-3,temp);
3087   }
3088   set_jump_target(done0, out);
3089   set_jump_target(done1, out);
3090   set_jump_target(done2, out);
3091   if(!c||!memtarget)
3092     add_stub_r(STORELR_STUB,jaddr,out,i,temp,i_regs,ccadj[i],reglist);
3093   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
3094     emit_addimm_no_flags(-ram_offset,temp);
3095     #if defined(HOST_IMM8)
3096     int ir=get_reg(i_regs->regmap,INVCP);
3097     assert(ir>=0);
3098     emit_cmpmem_indexedsr12_reg(ir,temp,1);
3099     #else
3100     emit_cmpmem_indexedsr12_imm(invalid_code,temp,1);
3101     #endif
3102     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3103     emit_callne(invalidate_addr_reg[temp]);
3104     #else
3105     void *jaddr2 = out;
3106     emit_jne(0);
3107     add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),temp,0,0,0);
3108     #endif
3109   }
3110 }
3111
3112 static void cop0_assemble(int i,struct regstat *i_regs)
3113 {
3114   if(opcode2[i]==0) // MFC0
3115   {
3116     signed char t=get_reg(i_regs->regmap,rt1[i]);
3117     u_int copr=(source[i]>>11)&0x1f;
3118     //assert(t>=0); // Why does this happen?  OOT is weird
3119     if(t>=0&&rt1[i]!=0) {
3120       emit_readword(&reg_cop0[copr],t);
3121     }
3122   }
3123   else if(opcode2[i]==4) // MTC0
3124   {
3125     signed char s=get_reg(i_regs->regmap,rs1[i]);
3126     char copr=(source[i]>>11)&0x1f;
3127     assert(s>=0);
3128     wb_register(rs1[i],i_regs->regmap,i_regs->dirty);
3129     if(copr==9||copr==11||copr==12||copr==13) {
3130       emit_readword(&last_count,HOST_TEMPREG);
3131       emit_loadreg(CCREG,HOST_CCREG); // TODO: do proper reg alloc
3132       emit_add(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
3133       emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3134       emit_writeword(HOST_CCREG,&Count);
3135     }
3136     // What a mess.  The status register (12) can enable interrupts,
3137     // so needs a special case to handle a pending interrupt.
3138     // The interrupt must be taken immediately, because a subsequent
3139     // instruction might disable interrupts again.
3140     if(copr==12||copr==13) {
3141       if (is_delayslot) {
3142         // burn cycles to cause cc_interrupt, which will
3143         // reschedule next_interupt. Relies on CCREG from above.
3144         assem_debug("MTC0 DS %d\n", copr);
3145         emit_writeword(HOST_CCREG,&last_count);
3146         emit_movimm(0,HOST_CCREG);
3147         emit_storereg(CCREG,HOST_CCREG);
3148         emit_loadreg(rs1[i],1);
3149         emit_movimm(copr,0);
3150         emit_far_call(pcsx_mtc0_ds);
3151         emit_loadreg(rs1[i],s);
3152         return;
3153       }
3154       emit_movimm(start+i*4+4,HOST_TEMPREG);
3155       emit_writeword(HOST_TEMPREG,&pcaddr);
3156       emit_movimm(0,HOST_TEMPREG);
3157       emit_writeword(HOST_TEMPREG,&pending_exception);
3158     }
3159     if(s==HOST_CCREG)
3160       emit_loadreg(rs1[i],1);
3161     else if(s!=1)
3162       emit_mov(s,1);
3163     emit_movimm(copr,0);
3164     emit_far_call(pcsx_mtc0);
3165     if(copr==9||copr==11||copr==12||copr==13) {
3166       emit_readword(&Count,HOST_CCREG);
3167       emit_readword(&next_interupt,HOST_TEMPREG);
3168       emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3169       emit_sub(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
3170       emit_writeword(HOST_TEMPREG,&last_count);
3171       emit_storereg(CCREG,HOST_CCREG);
3172     }
3173     if(copr==12||copr==13) {
3174       assert(!is_delayslot);
3175       emit_readword(&pending_exception,14);
3176       emit_test(14,14);
3177       void *jaddr = out;
3178       emit_jeq(0);
3179       emit_readword(&pcaddr, 0);
3180       emit_addimm(HOST_CCREG,2,HOST_CCREG);
3181       emit_far_call(get_addr_ht);
3182       emit_jmpreg(0);
3183       set_jump_target(jaddr, out);
3184     }
3185     emit_loadreg(rs1[i],s);
3186   }
3187   else
3188   {
3189     assert(opcode2[i]==0x10);
3190     //if((source[i]&0x3f)==0x10) // RFE
3191     {
3192       emit_readword(&Status,0);
3193       emit_andimm(0,0x3c,1);
3194       emit_andimm(0,~0xf,0);
3195       emit_orrshr_imm(1,2,0);
3196       emit_writeword(0,&Status);
3197     }
3198   }
3199 }
3200
3201 static void cop1_unusable(int i,struct regstat *i_regs)
3202 {
3203   // XXX: should just just do the exception instead
3204   //if(!cop1_usable)
3205   {
3206     void *jaddr=out;
3207     emit_jmp(0);
3208     add_stub_r(FP_STUB,jaddr,out,i,0,i_regs,is_delayslot,0);
3209   }
3210 }
3211
3212 static void cop1_assemble(int i,struct regstat *i_regs)
3213 {
3214   cop1_unusable(i, i_regs);
3215 }
3216
3217 static void c1ls_assemble(int i,struct regstat *i_regs)
3218 {
3219   cop1_unusable(i, i_regs);
3220 }
3221
3222 // FP_STUB
3223 static void do_cop1stub(int n)
3224 {
3225   literal_pool(256);
3226   assem_debug("do_cop1stub %x\n",start+stubs[n].a*4);
3227   set_jump_target(stubs[n].addr, out);
3228   int i=stubs[n].a;
3229 //  int rs=stubs[n].b;
3230   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3231   int ds=stubs[n].d;
3232   if(!ds) {
3233     load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
3234     //if(i_regs!=&regs[i]) printf("oops: regs[i]=%x i_regs=%x",(int)&regs[i],(int)i_regs);
3235   }
3236   //else {printf("fp exception in delay slot\n");}
3237   wb_dirtys(i_regs->regmap_entry,i_regs->wasdirty);
3238   if(regs[i].regmap_entry[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
3239   emit_movimm(start+(i-ds)*4,EAX); // Get PC
3240   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3241   emit_far_jump(ds?fp_exception_ds:fp_exception);
3242 }
3243
3244 static void cop2_get_dreg(u_int copr,signed char tl,signed char temp)
3245 {
3246   switch (copr) {
3247     case 1:
3248     case 3:
3249     case 5:
3250     case 8:
3251     case 9:
3252     case 10:
3253     case 11:
3254       emit_readword(&reg_cop2d[copr],tl);
3255       emit_signextend16(tl,tl);
3256       emit_writeword(tl,&reg_cop2d[copr]); // hmh
3257       break;
3258     case 7:
3259     case 16:
3260     case 17:
3261     case 18:
3262     case 19:
3263       emit_readword(&reg_cop2d[copr],tl);
3264       emit_andimm(tl,0xffff,tl);
3265       emit_writeword(tl,&reg_cop2d[copr]);
3266       break;
3267     case 15:
3268       emit_readword(&reg_cop2d[14],tl); // SXY2
3269       emit_writeword(tl,&reg_cop2d[copr]);
3270       break;
3271     case 28:
3272     case 29:
3273       c2op_mfc2_29_assemble(tl,temp);
3274       break;
3275     default:
3276       emit_readword(&reg_cop2d[copr],tl);
3277       break;
3278   }
3279 }
3280
3281 static void cop2_put_dreg(u_int copr,signed char sl,signed char temp)
3282 {
3283   switch (copr) {
3284     case 15:
3285       emit_readword(&reg_cop2d[13],temp);  // SXY1
3286       emit_writeword(sl,&reg_cop2d[copr]);
3287       emit_writeword(temp,&reg_cop2d[12]); // SXY0
3288       emit_readword(&reg_cop2d[14],temp);  // SXY2
3289       emit_writeword(sl,&reg_cop2d[14]);
3290       emit_writeword(temp,&reg_cop2d[13]); // SXY1
3291       break;
3292     case 28:
3293       emit_andimm(sl,0x001f,temp);
3294       emit_shlimm(temp,7,temp);
3295       emit_writeword(temp,&reg_cop2d[9]);
3296       emit_andimm(sl,0x03e0,temp);
3297       emit_shlimm(temp,2,temp);
3298       emit_writeword(temp,&reg_cop2d[10]);
3299       emit_andimm(sl,0x7c00,temp);
3300       emit_shrimm(temp,3,temp);
3301       emit_writeword(temp,&reg_cop2d[11]);
3302       emit_writeword(sl,&reg_cop2d[28]);
3303       break;
3304     case 30:
3305       emit_xorsar_imm(sl,sl,31,temp);
3306 #if defined(HAVE_ARMV5) || defined(__aarch64__)
3307       emit_clz(temp,temp);
3308 #else
3309       emit_movs(temp,HOST_TEMPREG);
3310       emit_movimm(0,temp);
3311       emit_jeq((int)out+4*4);
3312       emit_addpl_imm(temp,1,temp);
3313       emit_lslpls_imm(HOST_TEMPREG,1,HOST_TEMPREG);
3314       emit_jns((int)out-2*4);
3315 #endif
3316       emit_writeword(sl,&reg_cop2d[30]);
3317       emit_writeword(temp,&reg_cop2d[31]);
3318       break;
3319     case 31:
3320       break;
3321     default:
3322       emit_writeword(sl,&reg_cop2d[copr]);
3323       break;
3324   }
3325 }
3326
3327 static void c2ls_assemble(int i,struct regstat *i_regs)
3328 {
3329   int s,tl;
3330   int ar;
3331   int offset;
3332   int memtarget=0,c=0;
3333   void *jaddr2=NULL;
3334   enum stub_type type;
3335   int agr=AGEN1+(i&1);
3336   int fastio_reg_override=-1;
3337   u_int hr,reglist=0;
3338   u_int copr=(source[i]>>16)&0x1f;
3339   s=get_reg(i_regs->regmap,rs1[i]);
3340   tl=get_reg(i_regs->regmap,FTEMP);
3341   offset=imm[i];
3342   assert(rs1[i]>0);
3343   assert(tl>=0);
3344
3345   for(hr=0;hr<HOST_REGS;hr++) {
3346     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3347   }
3348   if(i_regs->regmap[HOST_CCREG]==CCREG)
3349     reglist&=~(1<<HOST_CCREG);
3350
3351   // get the address
3352   if (opcode[i]==0x3a) { // SWC2
3353     ar=get_reg(i_regs->regmap,agr);
3354     if(ar<0) ar=get_reg(i_regs->regmap,-1);
3355     reglist|=1<<ar;
3356   } else { // LWC2
3357     ar=tl;
3358   }
3359   if(s>=0) c=(i_regs->wasconst>>s)&1;
3360   memtarget=c&&(((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE);
3361   if (!offset&&!c&&s>=0) ar=s;
3362   assert(ar>=0);
3363
3364   if (opcode[i]==0x3a) { // SWC2
3365     cop2_get_dreg(copr,tl,-1);
3366     type=STOREW_STUB;
3367   }
3368   else
3369     type=LOADW_STUB;
3370
3371   if(c&&!memtarget) {
3372     jaddr2=out;
3373     emit_jmp(0); // inline_readstub/inline_writestub?
3374   }
3375   else {
3376     if(!c) {
3377       jaddr2=emit_fastpath_cmp_jump(i,ar,&fastio_reg_override);
3378     }
3379     else if(ram_offset&&memtarget) {
3380       host_tempreg_acquire();
3381       emit_addimm(ar,ram_offset,HOST_TEMPREG);
3382       fastio_reg_override=HOST_TEMPREG;
3383     }
3384     if (opcode[i]==0x32) { // LWC2
3385       int a=ar;
3386       if(fastio_reg_override>=0) a=fastio_reg_override;
3387       emit_readword_indexed(0,a,tl);
3388     }
3389     if (opcode[i]==0x3a) { // SWC2
3390       #ifdef DESTRUCTIVE_SHIFT
3391       if(!offset&&!c&&s>=0) emit_mov(s,ar);
3392       #endif
3393       int a=ar;
3394       if(fastio_reg_override>=0) a=fastio_reg_override;
3395       emit_writeword_indexed(tl,0,a);
3396     }
3397   }
3398   if(fastio_reg_override==HOST_TEMPREG)
3399     host_tempreg_release();
3400   if(jaddr2)
3401     add_stub_r(type,jaddr2,out,i,ar,i_regs,ccadj[i],reglist);
3402   if(opcode[i]==0x3a) // SWC2
3403   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
3404 #if defined(HOST_IMM8)
3405     int ir=get_reg(i_regs->regmap,INVCP);
3406     assert(ir>=0);
3407     emit_cmpmem_indexedsr12_reg(ir,ar,1);
3408 #else
3409     emit_cmpmem_indexedsr12_imm(invalid_code,ar,1);
3410 #endif
3411     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3412     emit_callne(invalidate_addr_reg[ar]);
3413     #else
3414     void *jaddr3 = out;
3415     emit_jne(0);
3416     add_stub(INVCODE_STUB,jaddr3,out,reglist|(1<<HOST_CCREG),ar,0,0,0);
3417     #endif
3418   }
3419   if (opcode[i]==0x32) { // LWC2
3420     host_tempreg_acquire();
3421     cop2_put_dreg(copr,tl,HOST_TEMPREG);
3422     host_tempreg_release();
3423   }
3424 }
3425
3426 static void cop2_assemble(int i,struct regstat *i_regs)
3427 {
3428   u_int copr=(source[i]>>11)&0x1f;
3429   signed char temp=get_reg(i_regs->regmap,-1);
3430   if (opcode2[i]==0) { // MFC2
3431     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3432     if(tl>=0&&rt1[i]!=0)
3433       cop2_get_dreg(copr,tl,temp);
3434   }
3435   else if (opcode2[i]==4) { // MTC2
3436     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3437     cop2_put_dreg(copr,sl,temp);
3438   }
3439   else if (opcode2[i]==2) // CFC2
3440   {
3441     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3442     if(tl>=0&&rt1[i]!=0)
3443       emit_readword(&reg_cop2c[copr],tl);
3444   }
3445   else if (opcode2[i]==6) // CTC2
3446   {
3447     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3448     switch(copr) {
3449       case 4:
3450       case 12:
3451       case 20:
3452       case 26:
3453       case 27:
3454       case 29:
3455       case 30:
3456         emit_signextend16(sl,temp);
3457         break;
3458       case 31:
3459         c2op_ctc2_31_assemble(sl,temp);
3460         break;
3461       default:
3462         temp=sl;
3463         break;
3464     }
3465     emit_writeword(temp,&reg_cop2c[copr]);
3466     assert(sl>=0);
3467   }
3468 }
3469
3470 static void do_unalignedwritestub(int n)
3471 {
3472   assem_debug("do_unalignedwritestub %x\n",start+stubs[n].a*4);
3473   literal_pool(256);
3474   set_jump_target(stubs[n].addr, out);
3475
3476   int i=stubs[n].a;
3477   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3478   int addr=stubs[n].b;
3479   u_int reglist=stubs[n].e;
3480   signed char *i_regmap=i_regs->regmap;
3481   int temp2=get_reg(i_regmap,FTEMP);
3482   int rt;
3483   rt=get_reg(i_regmap,rs2[i]);
3484   assert(rt>=0);
3485   assert(addr>=0);
3486   assert(opcode[i]==0x2a||opcode[i]==0x2e); // SWL/SWR only implemented
3487   reglist|=(1<<addr);
3488   reglist&=~(1<<temp2);
3489
3490 #if 1
3491   // don't bother with it and call write handler
3492   save_regs(reglist);
3493   pass_args(addr,rt);
3494   int cc=get_reg(i_regmap,CCREG);
3495   if(cc<0)
3496     emit_loadreg(CCREG,2);
3497   emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2);
3498   emit_far_call((opcode[i]==0x2a?jump_handle_swl:jump_handle_swr));
3499   emit_addimm(0,-CLOCK_ADJUST((int)stubs[n].d+1),cc<0?2:cc);
3500   if(cc<0)
3501     emit_storereg(CCREG,2);
3502   restore_regs(reglist);
3503   emit_jmp(stubs[n].retaddr); // return address
3504 #else
3505   emit_andimm(addr,0xfffffffc,temp2);
3506   emit_writeword(temp2,&address);
3507
3508   save_regs(reglist);
3509   emit_shrimm(addr,16,1);
3510   int cc=get_reg(i_regmap,CCREG);
3511   if(cc<0) {
3512     emit_loadreg(CCREG,2);
3513   }
3514   emit_movimm((u_int)readmem,0);
3515   emit_addimm(cc<0?2:cc,2*stubs[n].d+2,2);
3516   emit_call((int)&indirect_jump_indexed);
3517   restore_regs(reglist);
3518
3519   emit_readword(&readmem_dword,temp2);
3520   int temp=addr; //hmh
3521   emit_shlimm(addr,3,temp);
3522   emit_andimm(temp,24,temp);
3523   if (opcode[i]==0x2a) // SWL
3524     emit_xorimm(temp,24,temp);
3525   emit_movimm(-1,HOST_TEMPREG);
3526   if (opcode[i]==0x2a) { // SWL
3527     emit_bic_lsr(temp2,HOST_TEMPREG,temp,temp2);
3528     emit_orrshr(rt,temp,temp2);
3529   }else{
3530     emit_bic_lsl(temp2,HOST_TEMPREG,temp,temp2);
3531     emit_orrshl(rt,temp,temp2);
3532   }
3533   emit_readword(&address,addr);
3534   emit_writeword(temp2,&word);
3535   //save_regs(reglist); // don't need to, no state changes
3536   emit_shrimm(addr,16,1);
3537   emit_movimm((u_int)writemem,0);
3538   //emit_call((int)&indirect_jump_indexed);
3539   emit_mov(15,14);
3540   emit_readword_dualindexedx4(0,1,15);
3541   emit_readword(&Count,HOST_TEMPREG);
3542   emit_readword(&next_interupt,2);
3543   emit_addimm(HOST_TEMPREG,-2*stubs[n].d-2,HOST_TEMPREG);
3544   emit_writeword(2,&last_count);
3545   emit_sub(HOST_TEMPREG,2,cc<0?HOST_TEMPREG:cc);
3546   if(cc<0) {
3547     emit_storereg(CCREG,HOST_TEMPREG);
3548   }
3549   restore_regs(reglist);
3550   emit_jmp(stubs[n].retaddr); // return address
3551 #endif
3552 }
3553
3554 #ifndef multdiv_assemble
3555 void multdiv_assemble(int i,struct regstat *i_regs)
3556 {
3557   printf("Need multdiv_assemble for this architecture.\n");
3558   abort();
3559 }
3560 #endif
3561
3562 static void mov_assemble(int i,struct regstat *i_regs)
3563 {
3564   //if(opcode2[i]==0x10||opcode2[i]==0x12) { // MFHI/MFLO
3565   //if(opcode2[i]==0x11||opcode2[i]==0x13) { // MTHI/MTLO
3566   if(rt1[i]) {
3567     signed char sl,tl;
3568     tl=get_reg(i_regs->regmap,rt1[i]);
3569     //assert(tl>=0);
3570     if(tl>=0) {
3571       sl=get_reg(i_regs->regmap,rs1[i]);
3572       if(sl>=0) emit_mov(sl,tl);
3573       else emit_loadreg(rs1[i],tl);
3574     }
3575   }
3576 }
3577
3578 // call interpreter, exception handler, things that change pc/regs/cycles ...
3579 static void call_c_cpu_handler(int i, const struct regstat *i_regs, u_int pc, void *func)
3580 {
3581   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3582   assert(ccreg==HOST_CCREG);
3583   assert(!is_delayslot);
3584   (void)ccreg;
3585
3586   emit_movimm(pc,3); // Get PC
3587   emit_readword(&last_count,2);
3588   emit_writeword(3,&psxRegs.pc);
3589   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // XXX
3590   emit_add(2,HOST_CCREG,2);
3591   emit_writeword(2,&psxRegs.cycle);
3592   emit_far_call(func);
3593   emit_far_jump(jump_to_new_pc);
3594 }
3595
3596 static void syscall_assemble(int i,struct regstat *i_regs)
3597 {
3598   emit_movimm(0x20,0); // cause code
3599   emit_movimm(0,1);    // not in delay slot
3600   call_c_cpu_handler(i,i_regs,start+i*4,psxException);
3601 }
3602
3603 static void hlecall_assemble(int i,struct regstat *i_regs)
3604 {
3605   void *hlefunc = psxNULL;
3606   uint32_t hleCode = source[i] & 0x03ffffff;
3607   if (hleCode < ARRAY_SIZE(psxHLEt))
3608     hlefunc = psxHLEt[hleCode];
3609
3610   call_c_cpu_handler(i,i_regs,start+i*4+4,hlefunc);
3611 }
3612
3613 static void intcall_assemble(int i,struct regstat *i_regs)
3614 {
3615   call_c_cpu_handler(i,i_regs,start+i*4,execI);
3616 }
3617
3618 static void speculate_mov(int rs,int rt)
3619 {
3620   if(rt!=0) {
3621     smrv_strong_next|=1<<rt;
3622     smrv[rt]=smrv[rs];
3623   }
3624 }
3625
3626 static void speculate_mov_weak(int rs,int rt)
3627 {
3628   if(rt!=0) {
3629     smrv_weak_next|=1<<rt;
3630     smrv[rt]=smrv[rs];
3631   }
3632 }
3633
3634 static void speculate_register_values(int i)
3635 {
3636   if(i==0) {
3637     memcpy(smrv,psxRegs.GPR.r,sizeof(smrv));
3638     // gp,sp are likely to stay the same throughout the block
3639     smrv_strong_next=(1<<28)|(1<<29)|(1<<30);
3640     smrv_weak_next=~smrv_strong_next;
3641     //printf(" llr %08x\n", smrv[4]);
3642   }
3643   smrv_strong=smrv_strong_next;
3644   smrv_weak=smrv_weak_next;
3645   switch(itype[i]) {
3646     case ALU:
3647       if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3648       else if((smrv_strong>>rs2[i])&1) speculate_mov(rs2[i],rt1[i]);
3649       else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3650       else if((smrv_weak>>rs2[i])&1) speculate_mov_weak(rs2[i],rt1[i]);
3651       else {
3652         smrv_strong_next&=~(1<<rt1[i]);
3653         smrv_weak_next&=~(1<<rt1[i]);
3654       }
3655       break;
3656     case SHIFTIMM:
3657       smrv_strong_next&=~(1<<rt1[i]);
3658       smrv_weak_next&=~(1<<rt1[i]);
3659       // fallthrough
3660     case IMM16:
3661       if(rt1[i]&&is_const(&regs[i],rt1[i])) {
3662         int value,hr=get_reg(regs[i].regmap,rt1[i]);
3663         if(hr>=0) {
3664           if(get_final_value(hr,i,&value))
3665                smrv[rt1[i]]=value;
3666           else smrv[rt1[i]]=constmap[i][hr];
3667           smrv_strong_next|=1<<rt1[i];
3668         }
3669       }
3670       else {
3671         if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3672         else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3673       }
3674       break;
3675     case LOAD:
3676       if(start<0x2000&&(rt1[i]==26||(smrv[rt1[i]]>>24)==0xa0)) {
3677         // special case for BIOS
3678         smrv[rt1[i]]=0xa0000000;
3679         smrv_strong_next|=1<<rt1[i];
3680         break;
3681       }
3682       // fallthrough
3683     case SHIFT:
3684     case LOADLR:
3685     case MOV:
3686       smrv_strong_next&=~(1<<rt1[i]);
3687       smrv_weak_next&=~(1<<rt1[i]);
3688       break;
3689     case COP0:
3690     case COP2:
3691       if(opcode2[i]==0||opcode2[i]==2) { // MFC/CFC
3692         smrv_strong_next&=~(1<<rt1[i]);
3693         smrv_weak_next&=~(1<<rt1[i]);
3694       }
3695       break;
3696     case C2LS:
3697       if (opcode[i]==0x32) { // LWC2
3698         smrv_strong_next&=~(1<<rt1[i]);
3699         smrv_weak_next&=~(1<<rt1[i]);
3700       }
3701       break;
3702   }
3703 #if 0
3704   int r=4;
3705   printf("x %08x %08x %d %d c %08x %08x\n",smrv[r],start+i*4,
3706     ((smrv_strong>>r)&1),(smrv_weak>>r)&1,regs[i].isconst,regs[i].wasconst);
3707 #endif
3708 }
3709
3710 static void ds_assemble(int i,struct regstat *i_regs)
3711 {
3712   speculate_register_values(i);
3713   is_delayslot=1;
3714   switch(itype[i]) {
3715     case ALU:
3716       alu_assemble(i,i_regs);break;
3717     case IMM16:
3718       imm16_assemble(i,i_regs);break;
3719     case SHIFT:
3720       shift_assemble(i,i_regs);break;
3721     case SHIFTIMM:
3722       shiftimm_assemble(i,i_regs);break;
3723     case LOAD:
3724       load_assemble(i,i_regs);break;
3725     case LOADLR:
3726       loadlr_assemble(i,i_regs);break;
3727     case STORE:
3728       store_assemble(i,i_regs);break;
3729     case STORELR:
3730       storelr_assemble(i,i_regs);break;
3731     case COP0:
3732       cop0_assemble(i,i_regs);break;
3733     case COP1:
3734       cop1_assemble(i,i_regs);break;
3735     case C1LS:
3736       c1ls_assemble(i,i_regs);break;
3737     case COP2:
3738       cop2_assemble(i,i_regs);break;
3739     case C2LS:
3740       c2ls_assemble(i,i_regs);break;
3741     case C2OP:
3742       c2op_assemble(i,i_regs);break;
3743     case MULTDIV:
3744       multdiv_assemble(i,i_regs);break;
3745     case MOV:
3746       mov_assemble(i,i_regs);break;
3747     case SYSCALL:
3748     case HLECALL:
3749     case INTCALL:
3750     case SPAN:
3751     case UJUMP:
3752     case RJUMP:
3753     case CJUMP:
3754     case SJUMP:
3755       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
3756   }
3757   is_delayslot=0;
3758 }
3759
3760 // Is the branch target a valid internal jump?
3761 static int internal_branch(int addr)
3762 {
3763   if(addr&1) return 0; // Indirect (register) jump
3764   if(addr>=start && addr<start+slen*4-4)
3765   {
3766     return 1;
3767   }
3768   return 0;
3769 }
3770
3771 static void wb_invalidate(signed char pre[],signed char entry[],uint64_t dirty,uint64_t u)
3772 {
3773   int hr;
3774   for(hr=0;hr<HOST_REGS;hr++) {
3775     if(hr!=EXCLUDE_REG) {
3776       if(pre[hr]!=entry[hr]) {
3777         if(pre[hr]>=0) {
3778           if((dirty>>hr)&1) {
3779             if(get_reg(entry,pre[hr])<0) {
3780               assert(pre[hr]<64);
3781               if(!((u>>pre[hr])&1))
3782                 emit_storereg(pre[hr],hr);
3783             }
3784           }
3785         }
3786       }
3787     }
3788   }
3789   // Move from one register to another (no writeback)
3790   for(hr=0;hr<HOST_REGS;hr++) {
3791     if(hr!=EXCLUDE_REG) {
3792       if(pre[hr]!=entry[hr]) {
3793         if(pre[hr]>=0&&(pre[hr]&63)<TEMPREG) {
3794           int nr;
3795           if((nr=get_reg(entry,pre[hr]))>=0) {
3796             emit_mov(hr,nr);
3797           }
3798         }
3799       }
3800     }
3801   }
3802 }
3803
3804 // Load the specified registers
3805 // This only loads the registers given as arguments because
3806 // we don't want to load things that will be overwritten
3807 static void load_regs(signed char entry[],signed char regmap[],int rs1,int rs2)
3808 {
3809   int hr;
3810   // Load 32-bit regs
3811   for(hr=0;hr<HOST_REGS;hr++) {
3812     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3813       if(entry[hr]!=regmap[hr]) {
3814         if(regmap[hr]==rs1||regmap[hr]==rs2)
3815         {
3816           if(regmap[hr]==0) {
3817             emit_zeroreg(hr);
3818           }
3819           else
3820           {
3821             emit_loadreg(regmap[hr],hr);
3822           }
3823         }
3824       }
3825     }
3826   }
3827 }
3828
3829 // Load registers prior to the start of a loop
3830 // so that they are not loaded within the loop
3831 static void loop_preload(signed char pre[],signed char entry[])
3832 {
3833   int hr;
3834   for(hr=0;hr<HOST_REGS;hr++) {
3835     if(hr!=EXCLUDE_REG) {
3836       if(pre[hr]!=entry[hr]) {
3837         if(entry[hr]>=0) {
3838           if(get_reg(pre,entry[hr])<0) {
3839             assem_debug("loop preload:\n");
3840             //printf("loop preload: %d\n",hr);
3841             if(entry[hr]==0) {
3842               emit_zeroreg(hr);
3843             }
3844             else if(entry[hr]<TEMPREG)
3845             {
3846               emit_loadreg(entry[hr],hr);
3847             }
3848             else if(entry[hr]-64<TEMPREG)
3849             {
3850               emit_loadreg(entry[hr],hr);
3851             }
3852           }
3853         }
3854       }
3855     }
3856   }
3857 }
3858
3859 // Generate address for load/store instruction
3860 // goes to AGEN for writes, FTEMP for LOADLR and cop1/2 loads
3861 void address_generation(int i,struct regstat *i_regs,signed char entry[])
3862 {
3863   if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS||itype[i]==C2LS) {
3864     int ra=-1;
3865     int agr=AGEN1+(i&1);
3866     if(itype[i]==LOAD) {
3867       ra=get_reg(i_regs->regmap,rt1[i]);
3868       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3869       assert(ra>=0);
3870     }
3871     if(itype[i]==LOADLR) {
3872       ra=get_reg(i_regs->regmap,FTEMP);
3873     }
3874     if(itype[i]==STORE||itype[i]==STORELR) {
3875       ra=get_reg(i_regs->regmap,agr);
3876       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3877     }
3878     if(itype[i]==C1LS||itype[i]==C2LS) {
3879       if ((opcode[i]&0x3b)==0x31||(opcode[i]&0x3b)==0x32) // LWC1/LDC1/LWC2/LDC2
3880         ra=get_reg(i_regs->regmap,FTEMP);
3881       else { // SWC1/SDC1/SWC2/SDC2
3882         ra=get_reg(i_regs->regmap,agr);
3883         if(ra<0) ra=get_reg(i_regs->regmap,-1);
3884       }
3885     }
3886     int rs=get_reg(i_regs->regmap,rs1[i]);
3887     if(ra>=0) {
3888       int offset=imm[i];
3889       int c=(i_regs->wasconst>>rs)&1;
3890       if(rs1[i]==0) {
3891         // Using r0 as a base address
3892         if(!entry||entry[ra]!=agr) {
3893           if (opcode[i]==0x22||opcode[i]==0x26) {
3894             emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3895           }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3896             emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3897           }else{
3898             emit_movimm(offset,ra);
3899           }
3900         } // else did it in the previous cycle
3901       }
3902       else if(rs<0) {
3903         if(!entry||entry[ra]!=rs1[i])
3904           emit_loadreg(rs1[i],ra);
3905         //if(!entry||entry[ra]!=rs1[i])
3906         //  printf("poor load scheduling!\n");
3907       }
3908       else if(c) {
3909         if(rs1[i]!=rt1[i]||itype[i]!=LOAD) {
3910           if(!entry||entry[ra]!=agr) {
3911             if (opcode[i]==0x22||opcode[i]==0x26) {
3912               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3913             }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3914               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3915             }else{
3916               emit_movimm(constmap[i][rs]+offset,ra);
3917               regs[i].loadedconst|=1<<ra;
3918             }
3919           } // else did it in the previous cycle
3920         } // else load_consts already did it
3921       }
3922       if(offset&&!c&&rs1[i]) {
3923         if(rs>=0) {
3924           emit_addimm(rs,offset,ra);
3925         }else{
3926           emit_addimm(ra,offset,ra);
3927         }
3928       }
3929     }
3930   }
3931   // Preload constants for next instruction
3932   if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS||itype[i+1]==C2LS) {
3933     int agr,ra;
3934     // Actual address
3935     agr=AGEN1+((i+1)&1);
3936     ra=get_reg(i_regs->regmap,agr);
3937     if(ra>=0) {
3938       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
3939       int offset=imm[i+1];
3940       int c=(regs[i+1].wasconst>>rs)&1;
3941       if(c&&(rs1[i+1]!=rt1[i+1]||itype[i+1]!=LOAD)) {
3942         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3943           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3944         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3945           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3946         }else{
3947           emit_movimm(constmap[i+1][rs]+offset,ra);
3948           regs[i+1].loadedconst|=1<<ra;
3949         }
3950       }
3951       else if(rs1[i+1]==0) {
3952         // Using r0 as a base address
3953         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3954           emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3955         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3956           emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3957         }else{
3958           emit_movimm(offset,ra);
3959         }
3960       }
3961     }
3962   }
3963 }
3964
3965 static int get_final_value(int hr, int i, int *value)
3966 {
3967   int reg=regs[i].regmap[hr];
3968   while(i<slen-1) {
3969     if(regs[i+1].regmap[hr]!=reg) break;
3970     if(!((regs[i+1].isconst>>hr)&1)) break;
3971     if(bt[i+1]) break;
3972     i++;
3973   }
3974   if(i<slen-1) {
3975     if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
3976       *value=constmap[i][hr];
3977       return 1;
3978     }
3979     if(!bt[i+1]) {
3980       if(itype[i+1]==UJUMP||itype[i+1]==RJUMP||itype[i+1]==CJUMP||itype[i+1]==SJUMP) {
3981         // Load in delay slot, out-of-order execution
3982         if(itype[i+2]==LOAD&&rs1[i+2]==reg&&rt1[i+2]==reg&&((regs[i+1].wasconst>>hr)&1))
3983         {
3984           // Precompute load address
3985           *value=constmap[i][hr]+imm[i+2];
3986           return 1;
3987         }
3988       }
3989       if(itype[i+1]==LOAD&&rs1[i+1]==reg&&rt1[i+1]==reg)
3990       {
3991         // Precompute load address
3992         *value=constmap[i][hr]+imm[i+1];
3993         //printf("c=%x imm=%lx\n",(long)constmap[i][hr],imm[i+1]);
3994         return 1;
3995       }
3996     }
3997   }
3998   *value=constmap[i][hr];
3999   //printf("c=%lx\n",(long)constmap[i][hr]);
4000   if(i==slen-1) return 1;
4001   assert(reg < 64);
4002   return !((unneeded_reg[i+1]>>reg)&1);
4003 }
4004
4005 // Load registers with known constants
4006 static void load_consts(signed char pre[],signed char regmap[],int i)
4007 {
4008   int hr,hr2;
4009   // propagate loaded constant flags
4010   if(i==0||bt[i])
4011     regs[i].loadedconst=0;
4012   else {
4013     for(hr=0;hr<HOST_REGS;hr++) {
4014       if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((regs[i-1].isconst>>hr)&1)&&pre[hr]==regmap[hr]
4015          &&regmap[hr]==regs[i-1].regmap[hr]&&((regs[i-1].loadedconst>>hr)&1))
4016       {
4017         regs[i].loadedconst|=1<<hr;
4018       }
4019     }
4020   }
4021   // Load 32-bit regs
4022   for(hr=0;hr<HOST_REGS;hr++) {
4023     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
4024       //if(entry[hr]!=regmap[hr]) {
4025       if(!((regs[i].loadedconst>>hr)&1)) {
4026         assert(regmap[hr]<64);
4027         if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
4028           int value,similar=0;
4029           if(get_final_value(hr,i,&value)) {
4030             // see if some other register has similar value
4031             for(hr2=0;hr2<HOST_REGS;hr2++) {
4032               if(hr2!=EXCLUDE_REG&&((regs[i].loadedconst>>hr2)&1)) {
4033                 if(is_similar_value(value,constmap[i][hr2])) {
4034                   similar=1;
4035                   break;
4036                 }
4037               }
4038             }
4039             if(similar) {
4040               int value2;
4041               if(get_final_value(hr2,i,&value2)) // is this needed?
4042                 emit_movimm_from(value2,hr2,value,hr);
4043               else
4044                 emit_movimm(value,hr);
4045             }
4046             else if(value==0) {
4047               emit_zeroreg(hr);
4048             }
4049             else {
4050               emit_movimm(value,hr);
4051             }
4052           }
4053           regs[i].loadedconst|=1<<hr;
4054         }
4055       }
4056     }
4057   }
4058 }
4059
4060 void load_all_consts(signed char regmap[], u_int dirty, int i)
4061 {
4062   int hr;
4063   // Load 32-bit regs
4064   for(hr=0;hr<HOST_REGS;hr++) {
4065     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
4066       assert(regmap[hr] < 64);
4067       if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
4068         int value=constmap[i][hr];
4069         if(value==0) {
4070           emit_zeroreg(hr);
4071         }
4072         else {
4073           emit_movimm(value,hr);
4074         }
4075       }
4076     }
4077   }
4078 }
4079
4080 // Write out all dirty registers (except cycle count)
4081 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty)
4082 {
4083   int hr;
4084   for(hr=0;hr<HOST_REGS;hr++) {
4085     if(hr!=EXCLUDE_REG) {
4086       if(i_regmap[hr]>0) {
4087         if(i_regmap[hr]!=CCREG) {
4088           if((i_dirty>>hr)&1) {
4089             assert(i_regmap[hr]<64);
4090             emit_storereg(i_regmap[hr],hr);
4091           }
4092         }
4093       }
4094     }
4095   }
4096 }
4097
4098 // Write out dirty registers that we need to reload (pair with load_needed_regs)
4099 // This writes the registers not written by store_regs_bt
4100 void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr)
4101 {
4102   int hr;
4103   int t=(addr-start)>>2;
4104   for(hr=0;hr<HOST_REGS;hr++) {
4105     if(hr!=EXCLUDE_REG) {
4106       if(i_regmap[hr]>0) {
4107         if(i_regmap[hr]!=CCREG) {
4108           if(i_regmap[hr]==regs[t].regmap_entry[hr] && ((regs[t].dirty>>hr)&1)) {
4109             if((i_dirty>>hr)&1) {
4110               assert(i_regmap[hr]<64);
4111               emit_storereg(i_regmap[hr],hr);
4112             }
4113           }
4114         }
4115       }
4116     }
4117   }
4118 }
4119
4120 // Load all registers (except cycle count)
4121 void load_all_regs(signed char i_regmap[])
4122 {
4123   int hr;
4124   for(hr=0;hr<HOST_REGS;hr++) {
4125     if(hr!=EXCLUDE_REG) {
4126       if(i_regmap[hr]==0) {
4127         emit_zeroreg(hr);
4128       }
4129       else
4130       if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4131       {
4132         emit_loadreg(i_regmap[hr],hr);
4133       }
4134     }
4135   }
4136 }
4137
4138 // Load all current registers also needed by next instruction
4139 void load_needed_regs(signed char i_regmap[],signed char next_regmap[])
4140 {
4141   int hr;
4142   for(hr=0;hr<HOST_REGS;hr++) {
4143     if(hr!=EXCLUDE_REG) {
4144       if(get_reg(next_regmap,i_regmap[hr])>=0) {
4145         if(i_regmap[hr]==0) {
4146           emit_zeroreg(hr);
4147         }
4148         else
4149         if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4150         {
4151           emit_loadreg(i_regmap[hr],hr);
4152         }
4153       }
4154     }
4155   }
4156 }
4157
4158 // Load all regs, storing cycle count if necessary
4159 void load_regs_entry(int t)
4160 {
4161   int hr;
4162   if(is_ds[t]) emit_addimm(HOST_CCREG,CLOCK_ADJUST(1),HOST_CCREG);
4163   else if(ccadj[t]) emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[t]),HOST_CCREG);
4164   if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4165     emit_storereg(CCREG,HOST_CCREG);
4166   }
4167   // Load 32-bit regs
4168   for(hr=0;hr<HOST_REGS;hr++) {
4169     if(regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4170       if(regs[t].regmap_entry[hr]==0) {
4171         emit_zeroreg(hr);
4172       }
4173       else if(regs[t].regmap_entry[hr]!=CCREG)
4174       {
4175         emit_loadreg(regs[t].regmap_entry[hr],hr);
4176       }
4177     }
4178   }
4179 }
4180
4181 // Store dirty registers prior to branch
4182 void store_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4183 {
4184   if(internal_branch(addr))
4185   {
4186     int t=(addr-start)>>2;
4187     int hr;
4188     for(hr=0;hr<HOST_REGS;hr++) {
4189       if(hr!=EXCLUDE_REG) {
4190         if(i_regmap[hr]>0 && i_regmap[hr]!=CCREG) {
4191           if(i_regmap[hr]!=regs[t].regmap_entry[hr] || !((regs[t].dirty>>hr)&1)) {
4192             if((i_dirty>>hr)&1) {
4193               assert(i_regmap[hr]<64);
4194               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4195                 emit_storereg(i_regmap[hr],hr);
4196             }
4197           }
4198         }
4199       }
4200     }
4201   }
4202   else
4203   {
4204     // Branch out of this block, write out all dirty regs
4205     wb_dirtys(i_regmap,i_dirty);
4206   }
4207 }
4208
4209 // Load all needed registers for branch target
4210 static void load_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4211 {
4212   //if(addr>=start && addr<(start+slen*4))
4213   if(internal_branch(addr))
4214   {
4215     int t=(addr-start)>>2;
4216     int hr;
4217     // Store the cycle count before loading something else
4218     if(i_regmap[HOST_CCREG]!=CCREG) {
4219       assert(i_regmap[HOST_CCREG]==-1);
4220     }
4221     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4222       emit_storereg(CCREG,HOST_CCREG);
4223     }
4224     // Load 32-bit regs
4225     for(hr=0;hr<HOST_REGS;hr++) {
4226       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4227         if(i_regmap[hr]!=regs[t].regmap_entry[hr]) {
4228           if(regs[t].regmap_entry[hr]==0) {
4229             emit_zeroreg(hr);
4230           }
4231           else if(regs[t].regmap_entry[hr]!=CCREG)
4232           {
4233             emit_loadreg(regs[t].regmap_entry[hr],hr);
4234           }
4235         }
4236       }
4237     }
4238   }
4239 }
4240
4241 static int match_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
4242 {
4243   if(addr>=start && addr<start+slen*4-4)
4244   {
4245     int t=(addr-start)>>2;
4246     int hr;
4247     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) return 0;
4248     for(hr=0;hr<HOST_REGS;hr++)
4249     {
4250       if(hr!=EXCLUDE_REG)
4251       {
4252         if(i_regmap[hr]!=regs[t].regmap_entry[hr])
4253         {
4254           if(regs[t].regmap_entry[hr]>=0&&(regs[t].regmap_entry[hr]|64)<TEMPREG+64)
4255           {
4256             return 0;
4257           }
4258           else
4259           if((i_dirty>>hr)&1)
4260           {
4261             if(i_regmap[hr]<TEMPREG)
4262             {
4263               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4264                 return 0;
4265             }
4266             else if(i_regmap[hr]>=64&&i_regmap[hr]<TEMPREG+64)
4267             {
4268               assert(0);
4269             }
4270           }
4271         }
4272         else // Same register but is it 32-bit or dirty?
4273         if(i_regmap[hr]>=0)
4274         {
4275           if(!((regs[t].dirty>>hr)&1))
4276           {
4277             if((i_dirty>>hr)&1)
4278             {
4279               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4280               {
4281                 //printf("%x: dirty no match\n",addr);
4282                 return 0;
4283               }
4284             }
4285           }
4286         }
4287       }
4288     }
4289     // Delay slots are not valid branch targets
4290     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP)) return 0;
4291     // Delay slots require additional processing, so do not match
4292     if(is_ds[t]) return 0;
4293   }
4294   else
4295   {
4296     int hr;
4297     for(hr=0;hr<HOST_REGS;hr++)
4298     {
4299       if(hr!=EXCLUDE_REG)
4300       {
4301         if(i_regmap[hr]>=0)
4302         {
4303           if(hr!=HOST_CCREG||i_regmap[hr]!=CCREG)
4304           {
4305             if((i_dirty>>hr)&1)
4306             {
4307               return 0;
4308             }
4309           }
4310         }
4311       }
4312     }
4313   }
4314   return 1;
4315 }
4316
4317 #ifdef DRC_DBG
4318 static void drc_dbg_emit_do_cmp(int i)
4319 {
4320   extern void do_insn_cmp();
4321   //extern int cycle;
4322   u_int hr,reglist=0;
4323
4324   for(hr=0;hr<HOST_REGS;hr++)
4325     if(regs[i].regmap[hr]>=0) reglist|=1<<hr;
4326   save_regs(reglist);
4327   emit_movimm(start+i*4,0);
4328   emit_writeword(0,&pcaddr);
4329   emit_far_call(do_insn_cmp);
4330   //emit_readword(&cycle,0);
4331   //emit_addimm(0,2,0);
4332   //emit_writeword(0,&cycle);
4333   (void)get_reg2;
4334   restore_regs(reglist);
4335 }
4336 #else
4337 #define drc_dbg_emit_do_cmp(x)
4338 #endif
4339
4340 // Used when a branch jumps into the delay slot of another branch
4341 static void ds_assemble_entry(int i)
4342 {
4343   int t=(ba[i]-start)>>2;
4344   if (!instr_addr[t])
4345     instr_addr[t] = out;
4346   assem_debug("Assemble delay slot at %x\n",ba[i]);
4347   assem_debug("<->\n");
4348   drc_dbg_emit_do_cmp(t);
4349   if(regs[t].regmap_entry[HOST_CCREG]==CCREG&&regs[t].regmap[HOST_CCREG]!=CCREG)
4350     wb_register(CCREG,regs[t].regmap_entry,regs[t].wasdirty);
4351   load_regs(regs[t].regmap_entry,regs[t].regmap,rs1[t],rs2[t]);
4352   address_generation(t,&regs[t],regs[t].regmap_entry);
4353   if(itype[t]==STORE||itype[t]==STORELR||(opcode[t]&0x3b)==0x39||(opcode[t]&0x3b)==0x3a)
4354     load_regs(regs[t].regmap_entry,regs[t].regmap,INVCP,INVCP);
4355   is_delayslot=0;
4356   switch(itype[t]) {
4357     case ALU:
4358       alu_assemble(t,&regs[t]);break;
4359     case IMM16:
4360       imm16_assemble(t,&regs[t]);break;
4361     case SHIFT:
4362       shift_assemble(t,&regs[t]);break;
4363     case SHIFTIMM:
4364       shiftimm_assemble(t,&regs[t]);break;
4365     case LOAD:
4366       load_assemble(t,&regs[t]);break;
4367     case LOADLR:
4368       loadlr_assemble(t,&regs[t]);break;
4369     case STORE:
4370       store_assemble(t,&regs[t]);break;
4371     case STORELR:
4372       storelr_assemble(t,&regs[t]);break;
4373     case COP0:
4374       cop0_assemble(t,&regs[t]);break;
4375     case COP1:
4376       cop1_assemble(t,&regs[t]);break;
4377     case C1LS:
4378       c1ls_assemble(t,&regs[t]);break;
4379     case COP2:
4380       cop2_assemble(t,&regs[t]);break;
4381     case C2LS:
4382       c2ls_assemble(t,&regs[t]);break;
4383     case C2OP:
4384       c2op_assemble(t,&regs[t]);break;
4385     case MULTDIV:
4386       multdiv_assemble(t,&regs[t]);break;
4387     case MOV:
4388       mov_assemble(t,&regs[t]);break;
4389     case SYSCALL:
4390     case HLECALL:
4391     case INTCALL:
4392     case SPAN:
4393     case UJUMP:
4394     case RJUMP:
4395     case CJUMP:
4396     case SJUMP:
4397       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
4398   }
4399   store_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4400   load_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4401   if(internal_branch(ba[i]+4))
4402     assem_debug("branch: internal\n");
4403   else
4404     assem_debug("branch: external\n");
4405   assert(internal_branch(ba[i]+4));
4406   add_to_linker(out,ba[i]+4,internal_branch(ba[i]+4));
4407   emit_jmp(0);
4408 }
4409
4410 static void emit_extjump(void *addr, u_int target)
4411 {
4412   emit_extjump2(addr, target, dyna_linker);
4413 }
4414
4415 static void emit_extjump_ds(void *addr, u_int target)
4416 {
4417   emit_extjump2(addr, target, dyna_linker_ds);
4418 }
4419
4420 // Load 2 immediates optimizing for small code size
4421 static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2)
4422 {
4423   emit_movimm(imm1,rt1);
4424   emit_movimm_from(imm1,rt1,imm2,rt2);
4425 }
4426
4427 void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert)
4428 {
4429   int count;
4430   void *jaddr;
4431   void *idle=NULL;
4432   int t=0;
4433   if(itype[i]==RJUMP)
4434   {
4435     *adj=0;
4436   }
4437   //if(ba[i]>=start && ba[i]<(start+slen*4))
4438   if(internal_branch(ba[i]))
4439   {
4440     t=(ba[i]-start)>>2;
4441     if(is_ds[t]) *adj=-1; // Branch into delay slot adds an extra cycle
4442     else *adj=ccadj[t];
4443   }
4444   else
4445   {
4446     *adj=0;
4447   }
4448   count=ccadj[i];
4449   if(taken==TAKEN && i==(ba[i]-start)>>2 && source[i+1]==0) {
4450     // Idle loop
4451     if(count&1) emit_addimm_and_set_flags(2*(count+2),HOST_CCREG);
4452     idle=out;
4453     //emit_subfrommem(&idlecount,HOST_CCREG); // Count idle cycles
4454     emit_andimm(HOST_CCREG,3,HOST_CCREG);
4455     jaddr=out;
4456     emit_jmp(0);
4457   }
4458   else if(*adj==0||invert) {
4459     int cycles=CLOCK_ADJUST(count+2);
4460     // faster loop HACK
4461     if (t&&*adj) {
4462       int rel=t-i;
4463       if(-NO_CYCLE_PENALTY_THR<rel&&rel<0)
4464         cycles=CLOCK_ADJUST(*adj)+count+2-*adj;
4465     }
4466     emit_addimm_and_set_flags(cycles,HOST_CCREG);
4467     jaddr=out;
4468     emit_jns(0);
4469   }
4470   else
4471   {
4472     emit_cmpimm(HOST_CCREG,-CLOCK_ADJUST(count+2));
4473     jaddr=out;
4474     emit_jns(0);
4475   }
4476   add_stub(CC_STUB,jaddr,idle?idle:out,(*adj==0||invert||idle)?0:(count+2),i,addr,taken,0);
4477 }
4478
4479 static void do_ccstub(int n)
4480 {
4481   literal_pool(256);
4482   assem_debug("do_ccstub %x\n",start+(u_int)stubs[n].b*4);
4483   set_jump_target(stubs[n].addr, out);
4484   int i=stubs[n].b;
4485   if(stubs[n].d==NULLDS) {
4486     // Delay slot instruction is nullified ("likely" branch)
4487     wb_dirtys(regs[i].regmap,regs[i].dirty);
4488   }
4489   else if(stubs[n].d!=TAKEN) {
4490     wb_dirtys(branch_regs[i].regmap,branch_regs[i].dirty);
4491   }
4492   else {
4493     if(internal_branch(ba[i]))
4494       wb_needed_dirtys(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4495   }
4496   if(stubs[n].c!=-1)
4497   {
4498     // Save PC as return address
4499     emit_movimm(stubs[n].c,EAX);
4500     emit_writeword(EAX,&pcaddr);
4501   }
4502   else
4503   {
4504     // Return address depends on which way the branch goes
4505     if(itype[i]==CJUMP||itype[i]==SJUMP)
4506     {
4507       int s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4508       int s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4509       if(rs1[i]==0)
4510       {
4511         s1l=s2l;
4512         s2l=-1;
4513       }
4514       else if(rs2[i]==0)
4515       {
4516         s2l=-1;
4517       }
4518       assert(s1l>=0);
4519       #ifdef DESTRUCTIVE_WRITEBACK
4520       if(rs1[i]) {
4521         if((branch_regs[i].dirty>>s1l)&&1)
4522           emit_loadreg(rs1[i],s1l);
4523       }
4524       else {
4525         if((branch_regs[i].dirty>>s1l)&1)
4526           emit_loadreg(rs2[i],s1l);
4527       }
4528       if(s2l>=0)
4529         if((branch_regs[i].dirty>>s2l)&1)
4530           emit_loadreg(rs2[i],s2l);
4531       #endif
4532       int hr=0;
4533       int addr=-1,alt=-1,ntaddr=-1;
4534       while(hr<HOST_REGS)
4535       {
4536         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4537            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4538            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4539         {
4540           addr=hr++;break;
4541         }
4542         hr++;
4543       }
4544       while(hr<HOST_REGS)
4545       {
4546         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4547            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4548            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4549         {
4550           alt=hr++;break;
4551         }
4552         hr++;
4553       }
4554       if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
4555       {
4556         while(hr<HOST_REGS)
4557         {
4558           if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4559              (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4560              (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4561           {
4562             ntaddr=hr;break;
4563           }
4564           hr++;
4565         }
4566         assert(hr<HOST_REGS);
4567       }
4568       if((opcode[i]&0x2f)==4) // BEQ
4569       {
4570         #ifdef HAVE_CMOV_IMM
4571         if(s2l>=0) emit_cmp(s1l,s2l);
4572         else emit_test(s1l,s1l);
4573         emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
4574         #else
4575         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4576         if(s2l>=0) emit_cmp(s1l,s2l);
4577         else emit_test(s1l,s1l);
4578         emit_cmovne_reg(alt,addr);
4579         #endif
4580       }
4581       if((opcode[i]&0x2f)==5) // BNE
4582       {
4583         #ifdef HAVE_CMOV_IMM
4584         if(s2l>=0) emit_cmp(s1l,s2l);
4585         else emit_test(s1l,s1l);
4586         emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
4587         #else
4588         emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
4589         if(s2l>=0) emit_cmp(s1l,s2l);
4590         else emit_test(s1l,s1l);
4591         emit_cmovne_reg(alt,addr);
4592         #endif
4593       }
4594       if((opcode[i]&0x2f)==6) // BLEZ
4595       {
4596         //emit_movimm(ba[i],alt);
4597         //emit_movimm(start+i*4+8,addr);
4598         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4599         emit_cmpimm(s1l,1);
4600         emit_cmovl_reg(alt,addr);
4601       }
4602       if((opcode[i]&0x2f)==7) // BGTZ
4603       {
4604         //emit_movimm(ba[i],addr);
4605         //emit_movimm(start+i*4+8,ntaddr);
4606         emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
4607         emit_cmpimm(s1l,1);
4608         emit_cmovl_reg(ntaddr,addr);
4609       }
4610       if((opcode[i]==1)&&(opcode2[i]&0x2D)==0) // BLTZ
4611       {
4612         //emit_movimm(ba[i],alt);
4613         //emit_movimm(start+i*4+8,addr);
4614         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4615         emit_test(s1l,s1l);
4616         emit_cmovs_reg(alt,addr);
4617       }
4618       if((opcode[i]==1)&&(opcode2[i]&0x2D)==1) // BGEZ
4619       {
4620         //emit_movimm(ba[i],addr);
4621         //emit_movimm(start+i*4+8,alt);
4622         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4623         emit_test(s1l,s1l);
4624         emit_cmovs_reg(alt,addr);
4625       }
4626       if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
4627         if(source[i]&0x10000) // BC1T
4628         {
4629           //emit_movimm(ba[i],alt);
4630           //emit_movimm(start+i*4+8,addr);
4631           emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4632           emit_testimm(s1l,0x800000);
4633           emit_cmovne_reg(alt,addr);
4634         }
4635         else // BC1F
4636         {
4637           //emit_movimm(ba[i],addr);
4638           //emit_movimm(start+i*4+8,alt);
4639           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4640           emit_testimm(s1l,0x800000);
4641           emit_cmovne_reg(alt,addr);
4642         }
4643       }
4644       emit_writeword(addr,&pcaddr);
4645     }
4646     else
4647     if(itype[i]==RJUMP)
4648     {
4649       int r=get_reg(branch_regs[i].regmap,rs1[i]);
4650       if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4651         r=get_reg(branch_regs[i].regmap,RTEMP);
4652       }
4653       emit_writeword(r,&pcaddr);
4654     }
4655     else {SysPrintf("Unknown branch type in do_ccstub\n");abort();}
4656   }
4657   // Update cycle count
4658   assert(branch_regs[i].regmap[HOST_CCREG]==CCREG||branch_regs[i].regmap[HOST_CCREG]==-1);
4659   if(stubs[n].a) emit_addimm(HOST_CCREG,CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4660   emit_far_call(cc_interrupt);
4661   if(stubs[n].a) emit_addimm(HOST_CCREG,-CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4662   if(stubs[n].d==TAKEN) {
4663     if(internal_branch(ba[i]))
4664       load_needed_regs(branch_regs[i].regmap,regs[(ba[i]-start)>>2].regmap_entry);
4665     else if(itype[i]==RJUMP) {
4666       if(get_reg(branch_regs[i].regmap,RTEMP)>=0)
4667         emit_readword(&pcaddr,get_reg(branch_regs[i].regmap,RTEMP));
4668       else
4669         emit_loadreg(rs1[i],get_reg(branch_regs[i].regmap,rs1[i]));
4670     }
4671   }else if(stubs[n].d==NOTTAKEN) {
4672     if(i<slen-2) load_needed_regs(branch_regs[i].regmap,regmap_pre[i+2]);
4673     else load_all_regs(branch_regs[i].regmap);
4674   }else if(stubs[n].d==NULLDS) {
4675     // Delay slot instruction is nullified ("likely" branch)
4676     if(i<slen-2) load_needed_regs(regs[i].regmap,regmap_pre[i+2]);
4677     else load_all_regs(regs[i].regmap);
4678   }else{
4679     load_all_regs(branch_regs[i].regmap);
4680   }
4681   if (stubs[n].retaddr)
4682     emit_jmp(stubs[n].retaddr);
4683   else
4684     do_jump_vaddr(stubs[n].e);
4685 }
4686
4687 static void add_to_linker(void *addr, u_int target, int ext)
4688 {
4689   assert(linkcount < ARRAY_SIZE(link_addr));
4690   link_addr[linkcount].addr = addr;
4691   link_addr[linkcount].target = target;
4692   link_addr[linkcount].ext = ext;
4693   linkcount++;
4694 }
4695
4696 static void ujump_assemble_write_ra(int i)
4697 {
4698   int rt;
4699   unsigned int return_address;
4700   rt=get_reg(branch_regs[i].regmap,31);
4701   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4702   //assert(rt>=0);
4703   return_address=start+i*4+8;
4704   if(rt>=0) {
4705     #ifdef USE_MINI_HT
4706     if(internal_branch(return_address)&&rt1[i+1]!=31) {
4707       int temp=-1; // note: must be ds-safe
4708       #ifdef HOST_TEMPREG
4709       temp=HOST_TEMPREG;
4710       #endif
4711       if(temp>=0) do_miniht_insert(return_address,rt,temp);
4712       else emit_movimm(return_address,rt);
4713     }
4714     else
4715     #endif
4716     {
4717       #ifdef REG_PREFETCH
4718       if(temp>=0)
4719       {
4720         if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4721       }
4722       #endif
4723       emit_movimm(return_address,rt); // PC into link register
4724       #ifdef IMM_PREFETCH
4725       emit_prefetch(hash_table_get(return_address));
4726       #endif
4727     }
4728   }
4729 }
4730
4731 static void ujump_assemble(int i,struct regstat *i_regs)
4732 {
4733   int ra_done=0;
4734   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4735   address_generation(i+1,i_regs,regs[i].regmap_entry);
4736   #ifdef REG_PREFETCH
4737   int temp=get_reg(branch_regs[i].regmap,PTEMP);
4738   if(rt1[i]==31&&temp>=0)
4739   {
4740     signed char *i_regmap=i_regs->regmap;
4741     int return_address=start+i*4+8;
4742     if(get_reg(branch_regs[i].regmap,31)>0)
4743     if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4744   }
4745   #endif
4746   if(rt1[i]==31&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4747     ujump_assemble_write_ra(i); // writeback ra for DS
4748     ra_done=1;
4749   }
4750   ds_assemble(i+1,i_regs);
4751   uint64_t bc_unneeded=branch_regs[i].u;
4752   bc_unneeded|=1|(1LL<<rt1[i]);
4753   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4754   load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4755   if(!ra_done&&rt1[i]==31)
4756     ujump_assemble_write_ra(i);
4757   int cc,adj;
4758   cc=get_reg(branch_regs[i].regmap,CCREG);
4759   assert(cc==HOST_CCREG);
4760   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4761   #ifdef REG_PREFETCH
4762   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4763   #endif
4764   do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4765   if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4766   load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4767   if(internal_branch(ba[i]))
4768     assem_debug("branch: internal\n");
4769   else
4770     assem_debug("branch: external\n");
4771   if(internal_branch(ba[i])&&is_ds[(ba[i]-start)>>2]) {
4772     ds_assemble_entry(i);
4773   }
4774   else {
4775     add_to_linker(out,ba[i],internal_branch(ba[i]));
4776     emit_jmp(0);
4777   }
4778 }
4779
4780 static void rjump_assemble_write_ra(int i)
4781 {
4782   int rt,return_address;
4783   assert(rt1[i+1]!=rt1[i]);
4784   assert(rt2[i+1]!=rt1[i]);
4785   rt=get_reg(branch_regs[i].regmap,rt1[i]);
4786   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4787   assert(rt>=0);
4788   return_address=start+i*4+8;
4789   #ifdef REG_PREFETCH
4790   if(temp>=0)
4791   {
4792     if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4793   }
4794   #endif
4795   emit_movimm(return_address,rt); // PC into link register
4796   #ifdef IMM_PREFETCH
4797   emit_prefetch(hash_table_get(return_address));
4798   #endif
4799 }
4800
4801 static void rjump_assemble(int i,struct regstat *i_regs)
4802 {
4803   int temp;
4804   int rs,cc;
4805   int ra_done=0;
4806   rs=get_reg(branch_regs[i].regmap,rs1[i]);
4807   assert(rs>=0);
4808   if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4809     // Delay slot abuse, make a copy of the branch address register
4810     temp=get_reg(branch_regs[i].regmap,RTEMP);
4811     assert(temp>=0);
4812     assert(regs[i].regmap[temp]==RTEMP);
4813     emit_mov(rs,temp);
4814     rs=temp;
4815   }
4816   address_generation(i+1,i_regs,regs[i].regmap_entry);
4817   #ifdef REG_PREFETCH
4818   if(rt1[i]==31)
4819   {
4820     if((temp=get_reg(branch_regs[i].regmap,PTEMP))>=0) {
4821       signed char *i_regmap=i_regs->regmap;
4822       int return_address=start+i*4+8;
4823       if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4824     }
4825   }
4826   #endif
4827   #ifdef USE_MINI_HT
4828   if(rs1[i]==31) {
4829     int rh=get_reg(regs[i].regmap,RHASH);
4830     if(rh>=0) do_preload_rhash(rh);
4831   }
4832   #endif
4833   if(rt1[i]!=0&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4834     rjump_assemble_write_ra(i);
4835     ra_done=1;
4836   }
4837   ds_assemble(i+1,i_regs);
4838   uint64_t bc_unneeded=branch_regs[i].u;
4839   bc_unneeded|=1|(1LL<<rt1[i]);
4840   bc_unneeded&=~(1LL<<rs1[i]);
4841   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4842   load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],CCREG);
4843   if(!ra_done&&rt1[i]!=0)
4844     rjump_assemble_write_ra(i);
4845   cc=get_reg(branch_regs[i].regmap,CCREG);
4846   assert(cc==HOST_CCREG);
4847   (void)cc;
4848   #ifdef USE_MINI_HT
4849   int rh=get_reg(branch_regs[i].regmap,RHASH);
4850   int ht=get_reg(branch_regs[i].regmap,RHTBL);
4851   if(rs1[i]==31) {
4852     if(regs[i].regmap[rh]!=RHASH) do_preload_rhash(rh);
4853     do_preload_rhtbl(ht);
4854     do_rhash(rs,rh);
4855   }
4856   #endif
4857   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4858   #ifdef DESTRUCTIVE_WRITEBACK
4859   if((branch_regs[i].dirty>>rs)&1) {
4860     if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
4861       emit_loadreg(rs1[i],rs);
4862     }
4863   }
4864   #endif
4865   #ifdef REG_PREFETCH
4866   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4867   #endif
4868   #ifdef USE_MINI_HT
4869   if(rs1[i]==31) {
4870     do_miniht_load(ht,rh);
4871   }
4872   #endif
4873   //do_cc(i,branch_regs[i].regmap,&adj,-1,TAKEN);
4874   //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen
4875   //assert(adj==0);
4876   emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4877   add_stub(CC_STUB,out,NULL,0,i,-1,TAKEN,rs);
4878   if(itype[i+1]==COP0&&(source[i+1]&0x3f)==0x10)
4879     // special case for RFE
4880     emit_jmp(0);
4881   else
4882     emit_jns(0);
4883   //load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4884   #ifdef USE_MINI_HT
4885   if(rs1[i]==31) {
4886     do_miniht_jump(rs,rh,ht);
4887   }
4888   else
4889   #endif
4890   {
4891     do_jump_vaddr(rs);
4892   }
4893   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4894   if(rt1[i]!=31&&i<slen-2&&(((u_int)out)&7)) emit_mov(13,13);
4895   #endif
4896 }
4897
4898 static void cjump_assemble(int i,struct regstat *i_regs)
4899 {
4900   signed char *i_regmap=i_regs->regmap;
4901   int cc;
4902   int match;
4903   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4904   assem_debug("match=%d\n",match);
4905   int s1l,s2l;
4906   int unconditional=0,nop=0;
4907   int invert=0;
4908   int internal=internal_branch(ba[i]);
4909   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4910   if(!match) invert=1;
4911   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4912   if(i>(ba[i]-start)>>2) invert=1;
4913   #endif
4914   #ifdef __aarch64__
4915   invert=1; // because of near cond. branches
4916   #endif
4917
4918   if(ooo[i]) {
4919     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4920     s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4921   }
4922   else {
4923     s1l=get_reg(i_regmap,rs1[i]);
4924     s2l=get_reg(i_regmap,rs2[i]);
4925   }
4926   if(rs1[i]==0&&rs2[i]==0)
4927   {
4928     if(opcode[i]&1) nop=1;
4929     else unconditional=1;
4930     //assert(opcode[i]!=5);
4931     //assert(opcode[i]!=7);
4932     //assert(opcode[i]!=0x15);
4933     //assert(opcode[i]!=0x17);
4934   }
4935   else if(rs1[i]==0)
4936   {
4937     s1l=s2l;
4938     s2l=-1;
4939   }
4940   else if(rs2[i]==0)
4941   {
4942     s2l=-1;
4943   }
4944
4945   if(ooo[i]) {
4946     // Out of order execution (delay slot first)
4947     //printf("OOOE\n");
4948     address_generation(i+1,i_regs,regs[i].regmap_entry);
4949     ds_assemble(i+1,i_regs);
4950     int adj;
4951     uint64_t bc_unneeded=branch_regs[i].u;
4952     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
4953     bc_unneeded|=1;
4954     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4955     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs2[i]);
4956     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4957     cc=get_reg(branch_regs[i].regmap,CCREG);
4958     assert(cc==HOST_CCREG);
4959     if(unconditional)
4960       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4961     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
4962     //assem_debug("cycle count (adj)\n");
4963     if(unconditional) {
4964       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4965       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
4966         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4967         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4968         if(internal)
4969           assem_debug("branch: internal\n");
4970         else
4971           assem_debug("branch: external\n");
4972         if(internal&&is_ds[(ba[i]-start)>>2]) {
4973           ds_assemble_entry(i);
4974         }
4975         else {
4976           add_to_linker(out,ba[i],internal);
4977           emit_jmp(0);
4978         }
4979         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4980         if(((u_int)out)&7) emit_addnop(0);
4981         #endif
4982       }
4983     }
4984     else if(nop) {
4985       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4986       void *jaddr=out;
4987       emit_jns(0);
4988       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4989     }
4990     else {
4991       void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
4992       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
4993       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4994
4995       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4996       assert(s1l>=0);
4997       if(opcode[i]==4) // BEQ
4998       {
4999         if(s2l>=0) emit_cmp(s1l,s2l);
5000         else emit_test(s1l,s1l);
5001         if(invert){
5002           nottaken=out;
5003           emit_jne(DJT_1);
5004         }else{
5005           add_to_linker(out,ba[i],internal);
5006           emit_jeq(0);
5007         }
5008       }
5009       if(opcode[i]==5) // BNE
5010       {
5011         if(s2l>=0) emit_cmp(s1l,s2l);
5012         else emit_test(s1l,s1l);
5013         if(invert){
5014           nottaken=out;
5015           emit_jeq(DJT_1);
5016         }else{
5017           add_to_linker(out,ba[i],internal);
5018           emit_jne(0);
5019         }
5020       }
5021       if(opcode[i]==6) // BLEZ
5022       {
5023         emit_cmpimm(s1l,1);
5024         if(invert){
5025           nottaken=out;
5026           emit_jge(DJT_1);
5027         }else{
5028           add_to_linker(out,ba[i],internal);
5029           emit_jl(0);
5030         }
5031       }
5032       if(opcode[i]==7) // BGTZ
5033       {
5034         emit_cmpimm(s1l,1);
5035         if(invert){
5036           nottaken=out;
5037           emit_jl(DJT_1);
5038         }else{
5039           add_to_linker(out,ba[i],internal);
5040           emit_jge(0);
5041         }
5042       }
5043       if(invert) {
5044         if(taken) set_jump_target(taken, out);
5045         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5046         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5047           if(adj) {
5048             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5049             add_to_linker(out,ba[i],internal);
5050           }else{
5051             emit_addnop(13);
5052             add_to_linker(out,ba[i],internal*2);
5053           }
5054           emit_jmp(0);
5055         }else
5056         #endif
5057         {
5058           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5059           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5060           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5061           if(internal)
5062             assem_debug("branch: internal\n");
5063           else
5064             assem_debug("branch: external\n");
5065           if(internal&&is_ds[(ba[i]-start)>>2]) {
5066             ds_assemble_entry(i);
5067           }
5068           else {
5069             add_to_linker(out,ba[i],internal);
5070             emit_jmp(0);
5071           }
5072         }
5073         set_jump_target(nottaken, out);
5074       }
5075
5076       if(nottaken1) set_jump_target(nottaken1, out);
5077       if(adj) {
5078         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5079       }
5080     } // (!unconditional)
5081   } // if(ooo)
5082   else
5083   {
5084     // In-order execution (branch first)
5085     //if(likely[i]) printf("IOL\n");
5086     //else
5087     //printf("IOE\n");
5088     void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
5089     if(!unconditional&&!nop) {
5090       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5091       assert(s1l>=0);
5092       if((opcode[i]&0x2f)==4) // BEQ
5093       {
5094         if(s2l>=0) emit_cmp(s1l,s2l);
5095         else emit_test(s1l,s1l);
5096         nottaken=out;
5097         emit_jne(DJT_2);
5098       }
5099       if((opcode[i]&0x2f)==5) // BNE
5100       {
5101         if(s2l>=0) emit_cmp(s1l,s2l);
5102         else emit_test(s1l,s1l);
5103         nottaken=out;
5104         emit_jeq(DJT_2);
5105       }
5106       if((opcode[i]&0x2f)==6) // BLEZ
5107       {
5108         emit_cmpimm(s1l,1);
5109         nottaken=out;
5110         emit_jge(DJT_2);
5111       }
5112       if((opcode[i]&0x2f)==7) // BGTZ
5113       {
5114         emit_cmpimm(s1l,1);
5115         nottaken=out;
5116         emit_jl(DJT_2);
5117       }
5118     } // if(!unconditional)
5119     int adj;
5120     uint64_t ds_unneeded=branch_regs[i].u;
5121     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5122     ds_unneeded|=1;
5123     // branch taken
5124     if(!nop) {
5125       if(taken) set_jump_target(taken, out);
5126       assem_debug("1:\n");
5127       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5128       // load regs
5129       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5130       address_generation(i+1,&branch_regs[i],0);
5131       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5132       ds_assemble(i+1,&branch_regs[i]);
5133       cc=get_reg(branch_regs[i].regmap,CCREG);
5134       if(cc==-1) {
5135         emit_loadreg(CCREG,cc=HOST_CCREG);
5136         // CHECK: Is the following instruction (fall thru) allocated ok?
5137       }
5138       assert(cc==HOST_CCREG);
5139       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5140       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5141       assem_debug("cycle count (adj)\n");
5142       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5143       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5144       if(internal)
5145         assem_debug("branch: internal\n");
5146       else
5147         assem_debug("branch: external\n");
5148       if(internal&&is_ds[(ba[i]-start)>>2]) {
5149         ds_assemble_entry(i);
5150       }
5151       else {
5152         add_to_linker(out,ba[i],internal);
5153         emit_jmp(0);
5154       }
5155     }
5156     // branch not taken
5157     if(!unconditional) {
5158       if(nottaken1) set_jump_target(nottaken1, out);
5159       set_jump_target(nottaken, out);
5160       assem_debug("2:\n");
5161       if(!likely[i]) {
5162         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5163         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5164         address_generation(i+1,&branch_regs[i],0);
5165         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5166         ds_assemble(i+1,&branch_regs[i]);
5167       }
5168       cc=get_reg(branch_regs[i].regmap,CCREG);
5169       if(cc==-1&&!likely[i]) {
5170         // Cycle count isn't in a register, temporarily load it then write it out
5171         emit_loadreg(CCREG,HOST_CCREG);
5172         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5173         void *jaddr=out;
5174         emit_jns(0);
5175         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5176         emit_storereg(CCREG,HOST_CCREG);
5177       }
5178       else{
5179         cc=get_reg(i_regmap,CCREG);
5180         assert(cc==HOST_CCREG);
5181         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5182         void *jaddr=out;
5183         emit_jns(0);
5184         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5185       }
5186     }
5187   }
5188 }
5189
5190 static void sjump_assemble(int i,struct regstat *i_regs)
5191 {
5192   signed char *i_regmap=i_regs->regmap;
5193   int cc;
5194   int match;
5195   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5196   assem_debug("smatch=%d\n",match);
5197   int s1l;
5198   int unconditional=0,nevertaken=0;
5199   int invert=0;
5200   int internal=internal_branch(ba[i]);
5201   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
5202   if(!match) invert=1;
5203   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5204   if(i>(ba[i]-start)>>2) invert=1;
5205   #endif
5206   #ifdef __aarch64__
5207   invert=1; // because of near cond. branches
5208   #endif
5209
5210   //if(opcode2[i]>=0x10) return; // FIXME (BxxZAL)
5211   //assert(opcode2[i]<0x10||rs1[i]==0); // FIXME (BxxZAL)
5212
5213   if(ooo[i]) {
5214     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
5215   }
5216   else {
5217     s1l=get_reg(i_regmap,rs1[i]);
5218   }
5219   if(rs1[i]==0)
5220   {
5221     if(opcode2[i]&1) unconditional=1;
5222     else nevertaken=1;
5223     // These are never taken (r0 is never less than zero)
5224     //assert(opcode2[i]!=0);
5225     //assert(opcode2[i]!=2);
5226     //assert(opcode2[i]!=0x10);
5227     //assert(opcode2[i]!=0x12);
5228   }
5229
5230   if(ooo[i]) {
5231     // Out of order execution (delay slot first)
5232     //printf("OOOE\n");
5233     address_generation(i+1,i_regs,regs[i].regmap_entry);
5234     ds_assemble(i+1,i_regs);
5235     int adj;
5236     uint64_t bc_unneeded=branch_regs[i].u;
5237     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5238     bc_unneeded|=1;
5239     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
5240     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs1[i]);
5241     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5242     if(rt1[i]==31) {
5243       int rt,return_address;
5244       rt=get_reg(branch_regs[i].regmap,31);
5245       assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5246       if(rt>=0) {
5247         // Save the PC even if the branch is not taken
5248         return_address=start+i*4+8;
5249         emit_movimm(return_address,rt); // PC into link register
5250         #ifdef IMM_PREFETCH
5251         if(!nevertaken) emit_prefetch(hash_table_get(return_address));
5252         #endif
5253       }
5254     }
5255     cc=get_reg(branch_regs[i].regmap,CCREG);
5256     assert(cc==HOST_CCREG);
5257     if(unconditional)
5258       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5259     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
5260     assem_debug("cycle count (adj)\n");
5261     if(unconditional) {
5262       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5263       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
5264         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5265         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5266         if(internal)
5267           assem_debug("branch: internal\n");
5268         else
5269           assem_debug("branch: external\n");
5270         if(internal&&is_ds[(ba[i]-start)>>2]) {
5271           ds_assemble_entry(i);
5272         }
5273         else {
5274           add_to_linker(out,ba[i],internal);
5275           emit_jmp(0);
5276         }
5277         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5278         if(((u_int)out)&7) emit_addnop(0);
5279         #endif
5280       }
5281     }
5282     else if(nevertaken) {
5283       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5284       void *jaddr=out;
5285       emit_jns(0);
5286       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5287     }
5288     else {
5289       void *nottaken = NULL;
5290       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5291       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5292       {
5293         assert(s1l>=0);
5294         if((opcode2[i]&0xf)==0) // BLTZ/BLTZAL
5295         {
5296           emit_test(s1l,s1l);
5297           if(invert){
5298             nottaken=out;
5299             emit_jns(DJT_1);
5300           }else{
5301             add_to_linker(out,ba[i],internal);
5302             emit_js(0);
5303           }
5304         }
5305         if((opcode2[i]&0xf)==1) // BGEZ/BLTZAL
5306         {
5307           emit_test(s1l,s1l);
5308           if(invert){
5309             nottaken=out;
5310             emit_js(DJT_1);
5311           }else{
5312             add_to_linker(out,ba[i],internal);
5313             emit_jns(0);
5314           }
5315         }
5316       }
5317
5318       if(invert) {
5319         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5320         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5321           if(adj) {
5322             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5323             add_to_linker(out,ba[i],internal);
5324           }else{
5325             emit_addnop(13);
5326             add_to_linker(out,ba[i],internal*2);
5327           }
5328           emit_jmp(0);
5329         }else
5330         #endif
5331         {
5332           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5333           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5334           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5335           if(internal)
5336             assem_debug("branch: internal\n");
5337           else
5338             assem_debug("branch: external\n");
5339           if(internal&&is_ds[(ba[i]-start)>>2]) {
5340             ds_assemble_entry(i);
5341           }
5342           else {
5343             add_to_linker(out,ba[i],internal);
5344             emit_jmp(0);
5345           }
5346         }
5347         set_jump_target(nottaken, out);
5348       }
5349
5350       if(adj) {
5351         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5352       }
5353     } // (!unconditional)
5354   } // if(ooo)
5355   else
5356   {
5357     // In-order execution (branch first)
5358     //printf("IOE\n");
5359     void *nottaken = NULL;
5360     if(rt1[i]==31) {
5361       int rt,return_address;
5362       rt=get_reg(branch_regs[i].regmap,31);
5363       if(rt>=0) {
5364         // Save the PC even if the branch is not taken
5365         return_address=start+i*4+8;
5366         emit_movimm(return_address,rt); // PC into link register
5367         #ifdef IMM_PREFETCH
5368         emit_prefetch(hash_table_get(return_address));
5369         #endif
5370       }
5371     }
5372     if(!unconditional) {
5373       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5374         assert(s1l>=0);
5375         if((opcode2[i]&0x0d)==0) // BLTZ/BLTZL/BLTZAL/BLTZALL
5376         {
5377           emit_test(s1l,s1l);
5378           nottaken=out;
5379           emit_jns(DJT_1);
5380         }
5381         if((opcode2[i]&0x0d)==1) // BGEZ/BGEZL/BGEZAL/BGEZALL
5382         {
5383           emit_test(s1l,s1l);
5384           nottaken=out;
5385           emit_js(DJT_1);
5386         }
5387     } // if(!unconditional)
5388     int adj;
5389     uint64_t ds_unneeded=branch_regs[i].u;
5390     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5391     ds_unneeded|=1;
5392     // branch taken
5393     if(!nevertaken) {
5394       //assem_debug("1:\n");
5395       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5396       // load regs
5397       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5398       address_generation(i+1,&branch_regs[i],0);
5399       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5400       ds_assemble(i+1,&branch_regs[i]);
5401       cc=get_reg(branch_regs[i].regmap,CCREG);
5402       if(cc==-1) {
5403         emit_loadreg(CCREG,cc=HOST_CCREG);
5404         // CHECK: Is the following instruction (fall thru) allocated ok?
5405       }
5406       assert(cc==HOST_CCREG);
5407       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5408       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5409       assem_debug("cycle count (adj)\n");
5410       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5411       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5412       if(internal)
5413         assem_debug("branch: internal\n");
5414       else
5415         assem_debug("branch: external\n");
5416       if(internal&&is_ds[(ba[i]-start)>>2]) {
5417         ds_assemble_entry(i);
5418       }
5419       else {
5420         add_to_linker(out,ba[i],internal);
5421         emit_jmp(0);
5422       }
5423     }
5424     // branch not taken
5425     if(!unconditional) {
5426       set_jump_target(nottaken, out);
5427       assem_debug("1:\n");
5428       if(!likely[i]) {
5429         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5430         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5431         address_generation(i+1,&branch_regs[i],0);
5432         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5433         ds_assemble(i+1,&branch_regs[i]);
5434       }
5435       cc=get_reg(branch_regs[i].regmap,CCREG);
5436       if(cc==-1&&!likely[i]) {
5437         // Cycle count isn't in a register, temporarily load it then write it out
5438         emit_loadreg(CCREG,HOST_CCREG);
5439         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5440         void *jaddr=out;
5441         emit_jns(0);
5442         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5443         emit_storereg(CCREG,HOST_CCREG);
5444       }
5445       else{
5446         cc=get_reg(i_regmap,CCREG);
5447         assert(cc==HOST_CCREG);
5448         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5449         void *jaddr=out;
5450         emit_jns(0);
5451         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5452       }
5453     }
5454   }
5455 }
5456
5457 static void pagespan_assemble(int i,struct regstat *i_regs)
5458 {
5459   int s1l=get_reg(i_regs->regmap,rs1[i]);
5460   int s2l=get_reg(i_regs->regmap,rs2[i]);
5461   void *taken = NULL;
5462   void *nottaken = NULL;
5463   int unconditional=0;
5464   if(rs1[i]==0)
5465   {
5466     s1l=s2l;
5467     s2l=-1;
5468   }
5469   else if(rs2[i]==0)
5470   {
5471     s2l=-1;
5472   }
5473   int hr=0;
5474   int addr=-1,alt=-1,ntaddr=-1;
5475   if(i_regs->regmap[HOST_BTREG]<0) {addr=HOST_BTREG;}
5476   else {
5477     while(hr<HOST_REGS)
5478     {
5479       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
5480          (i_regs->regmap[hr]&63)!=rs1[i] &&
5481          (i_regs->regmap[hr]&63)!=rs2[i] )
5482       {
5483         addr=hr++;break;
5484       }
5485       hr++;
5486     }
5487   }
5488   while(hr<HOST_REGS)
5489   {
5490     if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5491        (i_regs->regmap[hr]&63)!=rs1[i] &&
5492        (i_regs->regmap[hr]&63)!=rs2[i] )
5493     {
5494       alt=hr++;break;
5495     }
5496     hr++;
5497   }
5498   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
5499   {
5500     while(hr<HOST_REGS)
5501     {
5502       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5503          (i_regs->regmap[hr]&63)!=rs1[i] &&
5504          (i_regs->regmap[hr]&63)!=rs2[i] )
5505       {
5506         ntaddr=hr;break;
5507       }
5508       hr++;
5509     }
5510   }
5511   assert(hr<HOST_REGS);
5512   if((opcode[i]&0x2e)==4||opcode[i]==0x11) { // BEQ/BNE/BEQL/BNEL/BC1
5513     load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
5514   }
5515   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5516   if(opcode[i]==2) // J
5517   {
5518     unconditional=1;
5519   }
5520   if(opcode[i]==3) // JAL
5521   {
5522     // TODO: mini_ht
5523     int rt=get_reg(i_regs->regmap,31);
5524     emit_movimm(start+i*4+8,rt);
5525     unconditional=1;
5526   }
5527   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
5528   {
5529     emit_mov(s1l,addr);
5530     if(opcode2[i]==9) // JALR
5531     {
5532       int rt=get_reg(i_regs->regmap,rt1[i]);
5533       emit_movimm(start+i*4+8,rt);
5534     }
5535   }
5536   if((opcode[i]&0x3f)==4) // BEQ
5537   {
5538     if(rs1[i]==rs2[i])
5539     {
5540       unconditional=1;
5541     }
5542     else
5543     #ifdef HAVE_CMOV_IMM
5544     if(1) {
5545       if(s2l>=0) emit_cmp(s1l,s2l);
5546       else emit_test(s1l,s1l);
5547       emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
5548     }
5549     else
5550     #endif
5551     {
5552       assert(s1l>=0);
5553       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5554       if(s2l>=0) emit_cmp(s1l,s2l);
5555       else emit_test(s1l,s1l);
5556       emit_cmovne_reg(alt,addr);
5557     }
5558   }
5559   if((opcode[i]&0x3f)==5) // BNE
5560   {
5561     #ifdef HAVE_CMOV_IMM
5562     if(s2l>=0) emit_cmp(s1l,s2l);
5563     else emit_test(s1l,s1l);
5564     emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
5565     #else
5566     assert(s1l>=0);
5567     emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
5568     if(s2l>=0) emit_cmp(s1l,s2l);
5569     else emit_test(s1l,s1l);
5570     emit_cmovne_reg(alt,addr);
5571     #endif
5572   }
5573   if((opcode[i]&0x3f)==0x14) // BEQL
5574   {
5575     if(s2l>=0) emit_cmp(s1l,s2l);
5576     else emit_test(s1l,s1l);
5577     if(nottaken) set_jump_target(nottaken, out);
5578     nottaken=out;
5579     emit_jne(0);
5580   }
5581   if((opcode[i]&0x3f)==0x15) // BNEL
5582   {
5583     if(s2l>=0) emit_cmp(s1l,s2l);
5584     else emit_test(s1l,s1l);
5585     nottaken=out;
5586     emit_jeq(0);
5587     if(taken) set_jump_target(taken, out);
5588   }
5589   if((opcode[i]&0x3f)==6) // BLEZ
5590   {
5591     emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5592     emit_cmpimm(s1l,1);
5593     emit_cmovl_reg(alt,addr);
5594   }
5595   if((opcode[i]&0x3f)==7) // BGTZ
5596   {
5597     emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
5598     emit_cmpimm(s1l,1);
5599     emit_cmovl_reg(ntaddr,addr);
5600   }
5601   if((opcode[i]&0x3f)==0x16) // BLEZL
5602   {
5603     assert((opcode[i]&0x3f)!=0x16);
5604   }
5605   if((opcode[i]&0x3f)==0x17) // BGTZL
5606   {
5607     assert((opcode[i]&0x3f)!=0x17);
5608   }
5609   assert(opcode[i]!=1); // BLTZ/BGEZ
5610
5611   //FIXME: Check CSREG
5612   if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
5613     if((source[i]&0x30000)==0) // BC1F
5614     {
5615       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5616       emit_testimm(s1l,0x800000);
5617       emit_cmovne_reg(alt,addr);
5618     }
5619     if((source[i]&0x30000)==0x10000) // BC1T
5620     {
5621       emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5622       emit_testimm(s1l,0x800000);
5623       emit_cmovne_reg(alt,addr);
5624     }
5625     if((source[i]&0x30000)==0x20000) // BC1FL
5626     {
5627       emit_testimm(s1l,0x800000);
5628       nottaken=out;
5629       emit_jne(0);
5630     }
5631     if((source[i]&0x30000)==0x30000) // BC1TL
5632     {
5633       emit_testimm(s1l,0x800000);
5634       nottaken=out;
5635       emit_jeq(0);
5636     }
5637   }
5638
5639   assert(i_regs->regmap[HOST_CCREG]==CCREG);
5640   wb_dirtys(regs[i].regmap,regs[i].dirty);
5641   if(likely[i]||unconditional)
5642   {
5643     emit_movimm(ba[i],HOST_BTREG);
5644   }
5645   else if(addr!=HOST_BTREG)
5646   {
5647     emit_mov(addr,HOST_BTREG);
5648   }
5649   void *branch_addr=out;
5650   emit_jmp(0);
5651   int target_addr=start+i*4+5;
5652   void *stub=out;
5653   void *compiled_target_addr=check_addr(target_addr);
5654   emit_extjump_ds(branch_addr, target_addr);
5655   if(compiled_target_addr) {
5656     set_jump_target(branch_addr, compiled_target_addr);
5657     add_link(target_addr,stub);
5658   }
5659   else set_jump_target(branch_addr, stub);
5660   if(likely[i]) {
5661     // Not-taken path
5662     set_jump_target(nottaken, out);
5663     wb_dirtys(regs[i].regmap,regs[i].dirty);
5664     void *branch_addr=out;
5665     emit_jmp(0);
5666     int target_addr=start+i*4+8;
5667     void *stub=out;
5668     void *compiled_target_addr=check_addr(target_addr);
5669     emit_extjump_ds(branch_addr, target_addr);
5670     if(compiled_target_addr) {
5671       set_jump_target(branch_addr, compiled_target_addr);
5672       add_link(target_addr,stub);
5673     }
5674     else set_jump_target(branch_addr, stub);
5675   }
5676 }
5677
5678 // Assemble the delay slot for the above
5679 static void pagespan_ds()
5680 {
5681   assem_debug("initial delay slot:\n");
5682   u_int vaddr=start+1;
5683   u_int page=get_page(vaddr);
5684   u_int vpage=get_vpage(vaddr);
5685   ll_add(jump_dirty+vpage,vaddr,(void *)out);
5686   do_dirty_stub_ds();
5687   ll_add(jump_in+page,vaddr,(void *)out);
5688   assert(regs[0].regmap_entry[HOST_CCREG]==CCREG);
5689   if(regs[0].regmap[HOST_CCREG]!=CCREG)
5690     wb_register(CCREG,regs[0].regmap_entry,regs[0].wasdirty);
5691   if(regs[0].regmap[HOST_BTREG]!=BTREG)
5692     emit_writeword(HOST_BTREG,&branch_target);
5693   load_regs(regs[0].regmap_entry,regs[0].regmap,rs1[0],rs2[0]);
5694   address_generation(0,&regs[0],regs[0].regmap_entry);
5695   if(itype[0]==STORE||itype[0]==STORELR||(opcode[0]&0x3b)==0x39||(opcode[0]&0x3b)==0x3a)
5696     load_regs(regs[0].regmap_entry,regs[0].regmap,INVCP,INVCP);
5697   is_delayslot=0;
5698   switch(itype[0]) {
5699     case ALU:
5700       alu_assemble(0,&regs[0]);break;
5701     case IMM16:
5702       imm16_assemble(0,&regs[0]);break;
5703     case SHIFT:
5704       shift_assemble(0,&regs[0]);break;
5705     case SHIFTIMM:
5706       shiftimm_assemble(0,&regs[0]);break;
5707     case LOAD:
5708       load_assemble(0,&regs[0]);break;
5709     case LOADLR:
5710       loadlr_assemble(0,&regs[0]);break;
5711     case STORE:
5712       store_assemble(0,&regs[0]);break;
5713     case STORELR:
5714       storelr_assemble(0,&regs[0]);break;
5715     case COP0:
5716       cop0_assemble(0,&regs[0]);break;
5717     case COP1:
5718       cop1_assemble(0,&regs[0]);break;
5719     case C1LS:
5720       c1ls_assemble(0,&regs[0]);break;
5721     case COP2:
5722       cop2_assemble(0,&regs[0]);break;
5723     case C2LS:
5724       c2ls_assemble(0,&regs[0]);break;
5725     case C2OP:
5726       c2op_assemble(0,&regs[0]);break;
5727     case MULTDIV:
5728       multdiv_assemble(0,&regs[0]);break;
5729     case MOV:
5730       mov_assemble(0,&regs[0]);break;
5731     case SYSCALL:
5732     case HLECALL:
5733     case INTCALL:
5734     case SPAN:
5735     case UJUMP:
5736     case RJUMP:
5737     case CJUMP:
5738     case SJUMP:
5739       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
5740   }
5741   int btaddr=get_reg(regs[0].regmap,BTREG);
5742   if(btaddr<0) {
5743     btaddr=get_reg(regs[0].regmap,-1);
5744     emit_readword(&branch_target,btaddr);
5745   }
5746   assert(btaddr!=HOST_CCREG);
5747   if(regs[0].regmap[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
5748 #ifdef HOST_IMM8
5749   host_tempreg_acquire();
5750   emit_movimm(start+4,HOST_TEMPREG);
5751   emit_cmp(btaddr,HOST_TEMPREG);
5752   host_tempreg_release();
5753 #else
5754   emit_cmpimm(btaddr,start+4);
5755 #endif
5756   void *branch = out;
5757   emit_jeq(0);
5758   store_regs_bt(regs[0].regmap,regs[0].dirty,-1);
5759   do_jump_vaddr(btaddr);
5760   set_jump_target(branch, out);
5761   store_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5762   load_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5763 }
5764
5765 // Basic liveness analysis for MIPS registers
5766 void unneeded_registers(int istart,int iend,int r)
5767 {
5768   int i;
5769   uint64_t u,gte_u,b,gte_b;
5770   uint64_t temp_u,temp_gte_u=0;
5771   uint64_t gte_u_unknown=0;
5772   if(new_dynarec_hacks&NDHACK_GTE_UNNEEDED)
5773     gte_u_unknown=~0ll;
5774   if(iend==slen-1) {
5775     u=1;
5776     gte_u=gte_u_unknown;
5777   }else{
5778     //u=unneeded_reg[iend+1];
5779     u=1;
5780     gte_u=gte_unneeded[iend+1];
5781   }
5782
5783   for (i=iend;i>=istart;i--)
5784   {
5785     //printf("unneeded registers i=%d (%d,%d) r=%d\n",i,istart,iend,r);
5786     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5787     {
5788       // If subroutine call, flag return address as a possible branch target
5789       if(rt1[i]==31 && i<slen-2) bt[i+2]=1;
5790
5791       if(ba[i]<start || ba[i]>=(start+slen*4))
5792       {
5793         // Branch out of this block, flush all regs
5794         u=1;
5795         gte_u=gte_u_unknown;
5796         branch_unneeded_reg[i]=u;
5797         // Merge in delay slot
5798         u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5799         u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5800         u|=1;
5801         gte_u|=gte_rt[i+1];
5802         gte_u&=~gte_rs[i+1];
5803         // If branch is "likely" (and conditional)
5804         // then we skip the delay slot on the fall-thru path
5805         if(likely[i]) {
5806           if(i<slen-1) {
5807             u&=unneeded_reg[i+2];
5808             gte_u&=gte_unneeded[i+2];
5809           }
5810           else
5811           {
5812             u=1;
5813             gte_u=gte_u_unknown;
5814           }
5815         }
5816       }
5817       else
5818       {
5819         // Internal branch, flag target
5820         bt[(ba[i]-start)>>2]=1;
5821         if(ba[i]<=start+i*4) {
5822           // Backward branch
5823           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5824           {
5825             // Unconditional branch
5826             temp_u=1;
5827             temp_gte_u=0;
5828           } else {
5829             // Conditional branch (not taken case)
5830             temp_u=unneeded_reg[i+2];
5831             temp_gte_u&=gte_unneeded[i+2];
5832           }
5833           // Merge in delay slot
5834           temp_u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5835           temp_u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5836           temp_u|=1;
5837           temp_gte_u|=gte_rt[i+1];
5838           temp_gte_u&=~gte_rs[i+1];
5839           // If branch is "likely" (and conditional)
5840           // then we skip the delay slot on the fall-thru path
5841           if(likely[i]) {
5842             if(i<slen-1) {
5843               temp_u&=unneeded_reg[i+2];
5844               temp_gte_u&=gte_unneeded[i+2];
5845             }
5846             else
5847             {
5848               temp_u=1;
5849               temp_gte_u=gte_u_unknown;
5850             }
5851           }
5852           temp_u|=(1LL<<rt1[i])|(1LL<<rt2[i]);
5853           temp_u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5854           temp_u|=1;
5855           temp_gte_u|=gte_rt[i];
5856           temp_gte_u&=~gte_rs[i];
5857           unneeded_reg[i]=temp_u;
5858           gte_unneeded[i]=temp_gte_u;
5859           // Only go three levels deep.  This recursion can take an
5860           // excessive amount of time if there are a lot of nested loops.
5861           if(r<2) {
5862             unneeded_registers((ba[i]-start)>>2,i-1,r+1);
5863           }else{
5864             unneeded_reg[(ba[i]-start)>>2]=1;
5865             gte_unneeded[(ba[i]-start)>>2]=gte_u_unknown;
5866           }
5867         } /*else*/ if(1) {
5868           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5869           {
5870             // Unconditional branch
5871             u=unneeded_reg[(ba[i]-start)>>2];
5872             gte_u=gte_unneeded[(ba[i]-start)>>2];
5873             branch_unneeded_reg[i]=u;
5874             // Merge in delay slot
5875             u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5876             u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5877             u|=1;
5878             gte_u|=gte_rt[i+1];
5879             gte_u&=~gte_rs[i+1];
5880           } else {
5881             // Conditional branch
5882             b=unneeded_reg[(ba[i]-start)>>2];
5883             gte_b=gte_unneeded[(ba[i]-start)>>2];
5884             branch_unneeded_reg[i]=b;
5885             // Branch delay slot
5886             b|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5887             b&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5888             b|=1;
5889             gte_b|=gte_rt[i+1];
5890             gte_b&=~gte_rs[i+1];
5891             // If branch is "likely" then we skip the
5892             // delay slot on the fall-thru path
5893             if(likely[i]) {
5894               u=b;
5895               gte_u=gte_b;
5896               if(i<slen-1) {
5897                 u&=unneeded_reg[i+2];
5898                 gte_u&=gte_unneeded[i+2];
5899               }
5900             } else {
5901               u&=b;
5902               gte_u&=gte_b;
5903             }
5904             if(i<slen-1) {
5905               branch_unneeded_reg[i]&=unneeded_reg[i+2];
5906             } else {
5907               branch_unneeded_reg[i]=1;
5908             }
5909           }
5910         }
5911       }
5912     }
5913     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5914     {
5915       // SYSCALL instruction (software interrupt)
5916       u=1;
5917     }
5918     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5919     {
5920       // ERET instruction (return from interrupt)
5921       u=1;
5922     }
5923     //u=1; // DEBUG
5924     // Written registers are unneeded
5925     u|=1LL<<rt1[i];
5926     u|=1LL<<rt2[i];
5927     gte_u|=gte_rt[i];
5928     // Accessed registers are needed
5929     u&=~(1LL<<rs1[i]);
5930     u&=~(1LL<<rs2[i]);
5931     gte_u&=~gte_rs[i];
5932     if(gte_rs[i]&&rt1[i]&&(unneeded_reg[i+1]&(1ll<<rt1[i])))
5933       gte_u|=gte_rs[i]&gte_unneeded[i+1]; // MFC2/CFC2 to dead register, unneeded
5934     // Source-target dependencies
5935     // R0 is always unneeded
5936     u|=1;
5937     // Save it
5938     unneeded_reg[i]=u;
5939     gte_unneeded[i]=gte_u;
5940     /*
5941     printf("ur (%d,%d) %x: ",istart,iend,start+i*4);
5942     printf("U:");
5943     int r;
5944     for(r=1;r<=CCREG;r++) {
5945       if((unneeded_reg[i]>>r)&1) {
5946         if(r==HIREG) printf(" HI");
5947         else if(r==LOREG) printf(" LO");
5948         else printf(" r%d",r);
5949       }
5950     }
5951     printf("\n");
5952     */
5953   }
5954 }
5955
5956 // Write back dirty registers as soon as we will no longer modify them,
5957 // so that we don't end up with lots of writes at the branches.
5958 void clean_registers(int istart,int iend,int wr)
5959 {
5960   int i;
5961   int r;
5962   u_int will_dirty_i,will_dirty_next,temp_will_dirty;
5963   u_int wont_dirty_i,wont_dirty_next,temp_wont_dirty;
5964   if(iend==slen-1) {
5965     will_dirty_i=will_dirty_next=0;
5966     wont_dirty_i=wont_dirty_next=0;
5967   }else{
5968     will_dirty_i=will_dirty_next=will_dirty[iend+1];
5969     wont_dirty_i=wont_dirty_next=wont_dirty[iend+1];
5970   }
5971   for (i=iend;i>=istart;i--)
5972   {
5973     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5974     {
5975       if(ba[i]<start || ba[i]>=(start+slen*4))
5976       {
5977         // Branch out of this block, flush all regs
5978         if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5979         {
5980           // Unconditional branch
5981           will_dirty_i=0;
5982           wont_dirty_i=0;
5983           // Merge in delay slot (will dirty)
5984           for(r=0;r<HOST_REGS;r++) {
5985             if(r!=EXCLUDE_REG) {
5986               if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5987               if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5988               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5989               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5990               if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5991               if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5992               if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5993               if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5994               if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5995               if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5996               if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5997               if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5998               if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5999               if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6000             }
6001           }
6002         }
6003         else
6004         {
6005           // Conditional branch
6006           will_dirty_i=0;
6007           wont_dirty_i=wont_dirty_next;
6008           // Merge in delay slot (will dirty)
6009           for(r=0;r<HOST_REGS;r++) {
6010             if(r!=EXCLUDE_REG) {
6011               if(!likely[i]) {
6012                 // Might not dirty if likely branch is not taken
6013                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6014                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6015                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6016                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6017                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6018                 if(branch_regs[i].regmap[r]==0) will_dirty_i&=~(1<<r);
6019                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6020                 //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6021                 //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6022                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6023                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6024                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6025                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6026                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6027               }
6028             }
6029           }
6030         }
6031         // Merge in delay slot (wont dirty)
6032         for(r=0;r<HOST_REGS;r++) {
6033           if(r!=EXCLUDE_REG) {
6034             if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6035             if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6036             if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6037             if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6038             if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6039             if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6040             if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6041             if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6042             if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6043             if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6044           }
6045         }
6046         if(wr) {
6047           #ifndef DESTRUCTIVE_WRITEBACK
6048           branch_regs[i].dirty&=wont_dirty_i;
6049           #endif
6050           branch_regs[i].dirty|=will_dirty_i;
6051         }
6052       }
6053       else
6054       {
6055         // Internal branch
6056         if(ba[i]<=start+i*4) {
6057           // Backward branch
6058           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
6059           {
6060             // Unconditional branch
6061             temp_will_dirty=0;
6062             temp_wont_dirty=0;
6063             // Merge in delay slot (will dirty)
6064             for(r=0;r<HOST_REGS;r++) {
6065               if(r!=EXCLUDE_REG) {
6066                 if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6067                 if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6068                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6069                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6070                 if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6071                 if(branch_regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6072                 if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6073                 if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6074                 if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6075                 if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6076                 if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6077                 if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6078                 if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6079                 if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6080               }
6081             }
6082           } else {
6083             // Conditional branch (not taken case)
6084             temp_will_dirty=will_dirty_next;
6085             temp_wont_dirty=wont_dirty_next;
6086             // Merge in delay slot (will dirty)
6087             for(r=0;r<HOST_REGS;r++) {
6088               if(r!=EXCLUDE_REG) {
6089                 if(!likely[i]) {
6090                   // Will not dirty if likely branch is not taken
6091                   if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6092                   if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6093                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6094                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6095                   if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6096                   if(branch_regs[i].regmap[r]==0) temp_will_dirty&=~(1<<r);
6097                   if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6098                   //if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
6099                   //if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
6100                   if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
6101                   if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
6102                   if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
6103                   if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
6104                   if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
6105                 }
6106               }
6107             }
6108           }
6109           // Merge in delay slot (wont dirty)
6110           for(r=0;r<HOST_REGS;r++) {
6111             if(r!=EXCLUDE_REG) {
6112               if((regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
6113               if((regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
6114               if((regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
6115               if((regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
6116               if(regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
6117               if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
6118               if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
6119               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
6120               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
6121               if(branch_regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
6122             }
6123           }
6124           // Deal with changed mappings
6125           if(i<iend) {
6126             for(r=0;r<HOST_REGS;r++) {
6127               if(r!=EXCLUDE_REG) {
6128                 if(regs[i].regmap[r]!=regmap_pre[i][r]) {
6129                   temp_will_dirty&=~(1<<r);
6130                   temp_wont_dirty&=~(1<<r);
6131                   if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6132                     temp_will_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6133                     temp_wont_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6134                   } else {
6135                     temp_will_dirty|=1<<r;
6136                     temp_wont_dirty|=1<<r;
6137                   }
6138                 }
6139               }
6140             }
6141           }
6142           if(wr) {
6143             will_dirty[i]=temp_will_dirty;
6144             wont_dirty[i]=temp_wont_dirty;
6145             clean_registers((ba[i]-start)>>2,i-1,0);
6146           }else{
6147             // Limit recursion.  It can take an excessive amount
6148             // of time if there are a lot of nested loops.
6149             will_dirty[(ba[i]-start)>>2]=0;
6150             wont_dirty[(ba[i]-start)>>2]=-1;
6151           }
6152         }
6153         /*else*/ if(1)
6154         {
6155           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
6156           {
6157             // Unconditional branch
6158             will_dirty_i=0;
6159             wont_dirty_i=0;
6160           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
6161             for(r=0;r<HOST_REGS;r++) {
6162               if(r!=EXCLUDE_REG) {
6163                 if(branch_regs[i].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6164                   will_dirty_i|=will_dirty[(ba[i]-start)>>2]&(1<<r);
6165                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6166                 }
6167                 if(branch_regs[i].regmap[r]>=0) {
6168                   will_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
6169                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
6170                 }
6171               }
6172             }
6173           //}
6174             // Merge in delay slot
6175             for(r=0;r<HOST_REGS;r++) {
6176               if(r!=EXCLUDE_REG) {
6177                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6178                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6179                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6180                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6181                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6182                 if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6183                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6184                 if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6185                 if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6186                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6187                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6188                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6189                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6190                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6191               }
6192             }
6193           } else {
6194             // Conditional branch
6195             will_dirty_i=will_dirty_next;
6196             wont_dirty_i=wont_dirty_next;
6197           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
6198             for(r=0;r<HOST_REGS;r++) {
6199               if(r!=EXCLUDE_REG) {
6200                 signed char target_reg=branch_regs[i].regmap[r];
6201                 if(target_reg==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6202                   will_dirty_i&=will_dirty[(ba[i]-start)>>2]&(1<<r);
6203                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6204                 }
6205                 else if(target_reg>=0) {
6206                   will_dirty_i&=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
6207                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
6208                 }
6209                 // Treat delay slot as part of branch too
6210                 /*if(regs[i+1].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
6211                   will_dirty[i+1]&=will_dirty[(ba[i]-start)>>2]&(1<<r);
6212                   wont_dirty[i+1]|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
6213                 }
6214                 else
6215                 {
6216                   will_dirty[i+1]&=~(1<<r);
6217                 }*/
6218               }
6219             }
6220           //}
6221             // Merge in delay slot
6222             for(r=0;r<HOST_REGS;r++) {
6223               if(r!=EXCLUDE_REG) {
6224                 if(!likely[i]) {
6225                   // Might not dirty if likely branch is not taken
6226                   if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6227                   if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6228                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6229                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6230                   if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6231                   if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6232                   if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6233                   //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6234                   //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6235                   if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
6236                   if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
6237                   if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6238                   if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6239                   if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6240                 }
6241               }
6242             }
6243           }
6244           // Merge in delay slot (won't dirty)
6245           for(r=0;r<HOST_REGS;r++) {
6246             if(r!=EXCLUDE_REG) {
6247               if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6248               if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6249               if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6250               if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6251               if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6252               if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6253               if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6254               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
6255               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
6256               if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6257             }
6258           }
6259           if(wr) {
6260             #ifndef DESTRUCTIVE_WRITEBACK
6261             branch_regs[i].dirty&=wont_dirty_i;
6262             #endif
6263             branch_regs[i].dirty|=will_dirty_i;
6264           }
6265         }
6266       }
6267     }
6268     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
6269     {
6270       // SYSCALL instruction (software interrupt)
6271       will_dirty_i=0;
6272       wont_dirty_i=0;
6273     }
6274     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
6275     {
6276       // ERET instruction (return from interrupt)
6277       will_dirty_i=0;
6278       wont_dirty_i=0;
6279     }
6280     will_dirty_next=will_dirty_i;
6281     wont_dirty_next=wont_dirty_i;
6282     for(r=0;r<HOST_REGS;r++) {
6283       if(r!=EXCLUDE_REG) {
6284         if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
6285         if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
6286         if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
6287         if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
6288         if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6289         if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6290         if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6291         if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6292         if(i>istart) {
6293           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP)
6294           {
6295             // Don't store a register immediately after writing it,
6296             // may prevent dual-issue.
6297             if((regs[i].regmap[r]&63)==rt1[i-1]) wont_dirty_i|=1<<r;
6298             if((regs[i].regmap[r]&63)==rt2[i-1]) wont_dirty_i|=1<<r;
6299           }
6300         }
6301       }
6302     }
6303     // Save it
6304     will_dirty[i]=will_dirty_i;
6305     wont_dirty[i]=wont_dirty_i;
6306     // Mark registers that won't be dirtied as not dirty
6307     if(wr) {
6308       /*printf("wr (%d,%d) %x will:",istart,iend,start+i*4);
6309       for(r=0;r<HOST_REGS;r++) {
6310         if((will_dirty_i>>r)&1) {
6311           printf(" r%d",r);
6312         }
6313       }
6314       printf("\n");*/
6315
6316       //if(i==istart||(itype[i-1]!=RJUMP&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP)) {
6317         regs[i].dirty|=will_dirty_i;
6318         #ifndef DESTRUCTIVE_WRITEBACK
6319         regs[i].dirty&=wont_dirty_i;
6320         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
6321         {
6322           if(i<iend-1&&itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
6323             for(r=0;r<HOST_REGS;r++) {
6324               if(r!=EXCLUDE_REG) {
6325                 if(regs[i].regmap[r]==regmap_pre[i+2][r]) {
6326                   regs[i+2].wasdirty&=wont_dirty_i|~(1<<r);
6327                 }else {/*printf("i: %x (%d) mismatch(+2): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6328               }
6329             }
6330           }
6331         }
6332         else
6333         {
6334           if(i<iend) {
6335             for(r=0;r<HOST_REGS;r++) {
6336               if(r!=EXCLUDE_REG) {
6337                 if(regs[i].regmap[r]==regmap_pre[i+1][r]) {
6338                   regs[i+1].wasdirty&=wont_dirty_i|~(1<<r);
6339                 }else {/*printf("i: %x (%d) mismatch(+1): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6340               }
6341             }
6342           }
6343         }
6344         #endif
6345       //}
6346     }
6347     // Deal with changed mappings
6348     temp_will_dirty=will_dirty_i;
6349     temp_wont_dirty=wont_dirty_i;
6350     for(r=0;r<HOST_REGS;r++) {
6351       if(r!=EXCLUDE_REG) {
6352         int nr;
6353         if(regs[i].regmap[r]==regmap_pre[i][r]) {
6354           if(wr) {
6355             #ifndef DESTRUCTIVE_WRITEBACK
6356             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6357             #endif
6358             regs[i].wasdirty|=will_dirty_i&(1<<r);
6359           }
6360         }
6361         else if(regmap_pre[i][r]>=0&&(nr=get_reg(regs[i].regmap,regmap_pre[i][r]))>=0) {
6362           // Register moved to a different register
6363           will_dirty_i&=~(1<<r);
6364           wont_dirty_i&=~(1<<r);
6365           will_dirty_i|=((temp_will_dirty>>nr)&1)<<r;
6366           wont_dirty_i|=((temp_wont_dirty>>nr)&1)<<r;
6367           if(wr) {
6368             #ifndef DESTRUCTIVE_WRITEBACK
6369             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6370             #endif
6371             regs[i].wasdirty|=will_dirty_i&(1<<r);
6372           }
6373         }
6374         else {
6375           will_dirty_i&=~(1<<r);
6376           wont_dirty_i&=~(1<<r);
6377           if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6378             will_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6379             wont_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6380           } else {
6381             wont_dirty_i|=1<<r;
6382             /*printf("i: %x (%d) mismatch: %d\n",start+i*4,i,r);assert(!((will_dirty>>r)&1));*/
6383           }
6384         }
6385       }
6386     }
6387   }
6388 }
6389
6390 #ifdef DISASM
6391   /* disassembly */
6392 void disassemble_inst(int i)
6393 {
6394     if (bt[i]) printf("*"); else printf(" ");
6395     switch(itype[i]) {
6396       case UJUMP:
6397         printf (" %x: %s %8x\n",start+i*4,insn[i],ba[i]);break;
6398       case CJUMP:
6399         printf (" %x: %s r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],i?start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14):*ba);break;
6400       case SJUMP:
6401         printf (" %x: %s r%d,%8x\n",start+i*4,insn[i],rs1[i],start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14));break;
6402       case RJUMP:
6403         if (opcode[i]==0x9&&rt1[i]!=31)
6404           printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i]);
6405         else
6406           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6407         break;
6408       case SPAN:
6409         printf (" %x: %s (pagespan) r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],ba[i]);break;
6410       case IMM16:
6411         if(opcode[i]==0xf) //LUI
6412           printf (" %x: %s r%d,%4x0000\n",start+i*4,insn[i],rt1[i],imm[i]&0xffff);
6413         else
6414           printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6415         break;
6416       case LOAD:
6417       case LOADLR:
6418         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6419         break;
6420       case STORE:
6421       case STORELR:
6422         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rs2[i],rs1[i],imm[i]);
6423         break;
6424       case ALU:
6425       case SHIFT:
6426         printf (" %x: %s r%d,r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i],rs2[i]);
6427         break;
6428       case MULTDIV:
6429         printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rs1[i],rs2[i]);
6430         break;
6431       case SHIFTIMM:
6432         printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6433         break;
6434       case MOV:
6435         if((opcode2[i]&0x1d)==0x10)
6436           printf (" %x: %s r%d\n",start+i*4,insn[i],rt1[i]);
6437         else if((opcode2[i]&0x1d)==0x11)
6438           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6439         else
6440           printf (" %x: %s\n",start+i*4,insn[i]);
6441         break;
6442       case COP0:
6443         if(opcode2[i]==0)
6444           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC0
6445         else if(opcode2[i]==4)
6446           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC0
6447         else printf (" %x: %s\n",start+i*4,insn[i]);
6448         break;
6449       case COP1:
6450         if(opcode2[i]<3)
6451           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC1
6452         else if(opcode2[i]>3)
6453           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC1
6454         else printf (" %x: %s\n",start+i*4,insn[i]);
6455         break;
6456       case COP2:
6457         if(opcode2[i]<3)
6458           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC2
6459         else if(opcode2[i]>3)
6460           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC2
6461         else printf (" %x: %s\n",start+i*4,insn[i]);
6462         break;
6463       case C1LS:
6464         printf (" %x: %s cpr1[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6465         break;
6466       case C2LS:
6467         printf (" %x: %s cpr2[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6468         break;
6469       case INTCALL:
6470         printf (" %x: %s (INTCALL)\n",start+i*4,insn[i]);
6471         break;
6472       default:
6473         //printf (" %s %8x\n",insn[i],source[i]);
6474         printf (" %x: %s\n",start+i*4,insn[i]);
6475     }
6476 }
6477 #else
6478 static void disassemble_inst(int i) {}
6479 #endif // DISASM
6480
6481 #define DRC_TEST_VAL 0x74657374
6482
6483 static void new_dynarec_test(void)
6484 {
6485   int (*testfunc)(void);
6486   void *beginning;
6487   int ret[2];
6488   size_t i;
6489
6490   // check structure linkage
6491   if ((u_char *)rcnts - (u_char *)&psxRegs != sizeof(psxRegs))
6492   {
6493     SysPrintf("linkage_arm* miscompilation/breakage detected.\n");
6494   }
6495
6496   SysPrintf("testing if we can run recompiled code...\n");
6497   ((volatile u_int *)out)[0]++; // make cache dirty
6498
6499   for (i = 0; i < ARRAY_SIZE(ret); i++) {
6500     out = ndrc->translation_cache;
6501     beginning = start_block();
6502     emit_movimm(DRC_TEST_VAL + i, 0); // test
6503     emit_ret();
6504     literal_pool(0);
6505     end_block(beginning);
6506     testfunc = beginning;
6507     ret[i] = testfunc();
6508   }
6509
6510   if (ret[0] == DRC_TEST_VAL && ret[1] == DRC_TEST_VAL + 1)
6511     SysPrintf("test passed.\n");
6512   else
6513     SysPrintf("test failed, will likely crash soon (r=%08x %08x)\n", ret[0], ret[1]);
6514   out = ndrc->translation_cache;
6515 }
6516
6517 // clear the state completely, instead of just marking
6518 // things invalid like invalidate_all_pages() does
6519 void new_dynarec_clear_full(void)
6520 {
6521   int n;
6522   out = ndrc->translation_cache;
6523   memset(invalid_code,1,sizeof(invalid_code));
6524   memset(hash_table,0xff,sizeof(hash_table));
6525   memset(mini_ht,-1,sizeof(mini_ht));
6526   memset(restore_candidate,0,sizeof(restore_candidate));
6527   memset(shadow,0,sizeof(shadow));
6528   copy=shadow;
6529   expirep=16384; // Expiry pointer, +2 blocks
6530   pending_exception=0;
6531   literalcount=0;
6532   stop_after_jal=0;
6533   inv_code_start=inv_code_end=~0;
6534   // TLB
6535   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6536   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6537   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6538 }
6539
6540 void new_dynarec_init(void)
6541 {
6542   SysPrintf("Init new dynarec\n");
6543
6544 #ifdef BASE_ADDR_DYNAMIC
6545   #ifdef VITA
6546   sceBlock = sceKernelAllocMemBlockForVM("code", 1 << TARGET_SIZE_2);
6547   if (sceBlock < 0)
6548     SysPrintf("sceKernelAllocMemBlockForVM failed\n");
6549   int ret = sceKernelGetMemBlockBase(sceBlock, (void **)&ndrc);
6550   if (ret < 0)
6551     SysPrintf("sceKernelGetMemBlockBase failed\n");
6552   #else
6553   uintptr_t desired_addr = 0;
6554   #ifdef __ELF__
6555   extern char _end;
6556   desired_addr = ((uintptr_t)&_end + 0xffffff) & ~0xffffffl;
6557   #endif
6558   ndrc = mmap((void *)desired_addr, sizeof(*ndrc),
6559             PROT_READ | PROT_WRITE | PROT_EXEC,
6560             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
6561   if (ndrc == MAP_FAILED) {
6562     SysPrintf("mmap() failed: %s\n", strerror(errno));
6563     abort();
6564   }
6565   #endif
6566 #else
6567   #ifndef NO_WRITE_EXEC
6568   // not all systems allow execute in data segment by default
6569   if (mprotect(ndrc, sizeof(ndrc->translation_cache) + sizeof(ndrc->tramp.ops),
6570                PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
6571     SysPrintf("mprotect() failed: %s\n", strerror(errno));
6572   #endif
6573 #endif
6574   out = ndrc->translation_cache;
6575   cycle_multiplier=200;
6576   new_dynarec_clear_full();
6577 #ifdef HOST_IMM8
6578   // Copy this into local area so we don't have to put it in every literal pool
6579   invc_ptr=invalid_code;
6580 #endif
6581   arch_init();
6582   new_dynarec_test();
6583 #ifndef RAM_FIXED
6584   ram_offset=(uintptr_t)rdram-0x80000000;
6585 #endif
6586   if (ram_offset!=0)
6587     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
6588 }
6589
6590 void new_dynarec_cleanup(void)
6591 {
6592   int n;
6593 #ifdef BASE_ADDR_DYNAMIC
6594   #ifdef VITA
6595   sceKernelFreeMemBlock(sceBlock);
6596   sceBlock = -1;
6597   #else
6598   if (munmap(ndrc, sizeof(*ndrc)) < 0)
6599     SysPrintf("munmap() failed\n");
6600   #endif
6601 #endif
6602   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6603   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6604   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6605   #ifdef ROM_COPY
6606   if (munmap (ROM_COPY, 67108864) < 0) {SysPrintf("munmap() failed\n");}
6607   #endif
6608 }
6609
6610 static u_int *get_source_start(u_int addr, u_int *limit)
6611 {
6612   if (addr < 0x00200000 ||
6613     (0xa0000000 <= addr && addr < 0xa0200000)) {
6614     // used for BIOS calls mostly?
6615     *limit = (addr&0xa0000000)|0x00200000;
6616     return (u_int *)(rdram + (addr&0x1fffff));
6617   }
6618   else if (!Config.HLE && (
6619     /* (0x9fc00000 <= addr && addr < 0x9fc80000) ||*/
6620     (0xbfc00000 <= addr && addr < 0xbfc80000))) {
6621     // BIOS
6622     *limit = (addr & 0xfff00000) | 0x80000;
6623     return (u_int *)((u_char *)psxR + (addr&0x7ffff));
6624   }
6625   else if (addr >= 0x80000000 && addr < 0x80000000+RAM_SIZE) {
6626     *limit = (addr & 0x80600000) + 0x00200000;
6627     return (u_int *)(rdram + (addr&0x1fffff));
6628   }
6629   return NULL;
6630 }
6631
6632 static u_int scan_for_ret(u_int addr)
6633 {
6634   u_int limit = 0;
6635   u_int *mem;
6636
6637   mem = get_source_start(addr, &limit);
6638   if (mem == NULL)
6639     return addr;
6640
6641   if (limit > addr + 0x1000)
6642     limit = addr + 0x1000;
6643   for (; addr < limit; addr += 4, mem++) {
6644     if (*mem == 0x03e00008) // jr $ra
6645       return addr + 8;
6646   }
6647   return addr;
6648 }
6649
6650 struct savestate_block {
6651   uint32_t addr;
6652   uint32_t regflags;
6653 };
6654
6655 static int addr_cmp(const void *p1_, const void *p2_)
6656 {
6657   const struct savestate_block *p1 = p1_, *p2 = p2_;
6658   return p1->addr - p2->addr;
6659 }
6660
6661 int new_dynarec_save_blocks(void *save, int size)
6662 {
6663   struct savestate_block *blocks = save;
6664   int maxcount = size / sizeof(blocks[0]);
6665   struct savestate_block tmp_blocks[1024];
6666   struct ll_entry *head;
6667   int p, s, d, o, bcnt;
6668   u_int addr;
6669
6670   o = 0;
6671   for (p = 0; p < ARRAY_SIZE(jump_in); p++) {
6672     bcnt = 0;
6673     for (head = jump_in[p]; head != NULL; head = head->next) {
6674       tmp_blocks[bcnt].addr = head->vaddr;
6675       tmp_blocks[bcnt].regflags = head->reg_sv_flags;
6676       bcnt++;
6677     }
6678     if (bcnt < 1)
6679       continue;
6680     qsort(tmp_blocks, bcnt, sizeof(tmp_blocks[0]), addr_cmp);
6681
6682     addr = tmp_blocks[0].addr;
6683     for (s = d = 0; s < bcnt; s++) {
6684       if (tmp_blocks[s].addr < addr)
6685         continue;
6686       if (d == 0 || tmp_blocks[d-1].addr != tmp_blocks[s].addr)
6687         tmp_blocks[d++] = tmp_blocks[s];
6688       addr = scan_for_ret(tmp_blocks[s].addr);
6689     }
6690
6691     if (o + d > maxcount)
6692       d = maxcount - o;
6693     memcpy(&blocks[o], tmp_blocks, d * sizeof(blocks[0]));
6694     o += d;
6695   }
6696
6697   return o * sizeof(blocks[0]);
6698 }
6699
6700 void new_dynarec_load_blocks(const void *save, int size)
6701 {
6702   const struct savestate_block *blocks = save;
6703   int count = size / sizeof(blocks[0]);
6704   u_int regs_save[32];
6705   uint32_t f;
6706   int i, b;
6707
6708   get_addr(psxRegs.pc);
6709
6710   // change GPRs for speculation to at least partially work..
6711   memcpy(regs_save, &psxRegs.GPR, sizeof(regs_save));
6712   for (i = 1; i < 32; i++)
6713     psxRegs.GPR.r[i] = 0x80000000;
6714
6715   for (b = 0; b < count; b++) {
6716     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6717       if (f & 1)
6718         psxRegs.GPR.r[i] = 0x1f800000;
6719     }
6720
6721     get_addr(blocks[b].addr);
6722
6723     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6724       if (f & 1)
6725         psxRegs.GPR.r[i] = 0x80000000;
6726     }
6727   }
6728
6729   memcpy(&psxRegs.GPR, regs_save, sizeof(regs_save));
6730 }
6731
6732 int new_recompile_block(u_int addr)
6733 {
6734   u_int pagelimit = 0;
6735   u_int state_rflags = 0;
6736   int i;
6737
6738   assem_debug("NOTCOMPILED: addr = %x -> %p\n", addr, out);
6739   //printf("TRACE: count=%d next=%d (compile %x)\n",Count,next_interupt,addr);
6740   //if(debug)
6741   //printf("fpu mapping=%x enabled=%x\n",(Status & 0x04000000)>>26,(Status & 0x20000000)>>29);
6742
6743   // this is just for speculation
6744   for (i = 1; i < 32; i++) {
6745     if ((psxRegs.GPR.r[i] & 0xffff0000) == 0x1f800000)
6746       state_rflags |= 1 << i;
6747   }
6748
6749   start = (u_int)addr&~3;
6750   //assert(((u_int)addr&1)==0); // start-in-delay-slot flag
6751   new_dynarec_did_compile=1;
6752   if (Config.HLE && start == 0x80001000) // hlecall
6753   {
6754     // XXX: is this enough? Maybe check hleSoftCall?
6755     void *beginning=start_block();
6756     u_int page=get_page(start);
6757
6758     invalid_code[start>>12]=0;
6759     emit_movimm(start,0);
6760     emit_writeword(0,&pcaddr);
6761     emit_far_jump(new_dyna_leave);
6762     literal_pool(0);
6763     end_block(beginning);
6764     ll_add_flags(jump_in+page,start,state_rflags,(void *)beginning);
6765     return 0;
6766   }
6767
6768   source = get_source_start(start, &pagelimit);
6769   if (source == NULL) {
6770     SysPrintf("Compile at bogus memory address: %08x\n", addr);
6771     abort();
6772   }
6773
6774   /* Pass 1: disassemble */
6775   /* Pass 2: register dependencies, branch targets */
6776   /* Pass 3: register allocation */
6777   /* Pass 4: branch dependencies */
6778   /* Pass 5: pre-alloc */
6779   /* Pass 6: optimize clean/dirty state */
6780   /* Pass 7: flag 32-bit registers */
6781   /* Pass 8: assembly */
6782   /* Pass 9: linker */
6783   /* Pass 10: garbage collection / free memory */
6784
6785   int j;
6786   int done=0;
6787   unsigned int type,op,op2;
6788
6789   //printf("addr = %x source = %x %x\n", addr,source,source[0]);
6790
6791   /* Pass 1 disassembly */
6792
6793   for(i=0;!done;i++) {
6794     bt[i]=0;likely[i]=0;ooo[i]=0;op2=0;
6795     minimum_free_regs[i]=0;
6796     opcode[i]=op=source[i]>>26;
6797     switch(op)
6798     {
6799       case 0x00: strcpy(insn[i],"special"); type=NI;
6800         op2=source[i]&0x3f;
6801         switch(op2)
6802         {
6803           case 0x00: strcpy(insn[i],"SLL"); type=SHIFTIMM; break;
6804           case 0x02: strcpy(insn[i],"SRL"); type=SHIFTIMM; break;
6805           case 0x03: strcpy(insn[i],"SRA"); type=SHIFTIMM; break;
6806           case 0x04: strcpy(insn[i],"SLLV"); type=SHIFT; break;
6807           case 0x06: strcpy(insn[i],"SRLV"); type=SHIFT; break;
6808           case 0x07: strcpy(insn[i],"SRAV"); type=SHIFT; break;
6809           case 0x08: strcpy(insn[i],"JR"); type=RJUMP; break;
6810           case 0x09: strcpy(insn[i],"JALR"); type=RJUMP; break;
6811           case 0x0C: strcpy(insn[i],"SYSCALL"); type=SYSCALL; break;
6812           case 0x0D: strcpy(insn[i],"BREAK"); type=OTHER; break;
6813           case 0x0F: strcpy(insn[i],"SYNC"); type=OTHER; break;
6814           case 0x10: strcpy(insn[i],"MFHI"); type=MOV; break;
6815           case 0x11: strcpy(insn[i],"MTHI"); type=MOV; break;
6816           case 0x12: strcpy(insn[i],"MFLO"); type=MOV; break;
6817           case 0x13: strcpy(insn[i],"MTLO"); type=MOV; break;
6818           case 0x18: strcpy(insn[i],"MULT"); type=MULTDIV; break;
6819           case 0x19: strcpy(insn[i],"MULTU"); type=MULTDIV; break;
6820           case 0x1A: strcpy(insn[i],"DIV"); type=MULTDIV; break;
6821           case 0x1B: strcpy(insn[i],"DIVU"); type=MULTDIV; break;
6822           case 0x20: strcpy(insn[i],"ADD"); type=ALU; break;
6823           case 0x21: strcpy(insn[i],"ADDU"); type=ALU; break;
6824           case 0x22: strcpy(insn[i],"SUB"); type=ALU; break;
6825           case 0x23: strcpy(insn[i],"SUBU"); type=ALU; break;
6826           case 0x24: strcpy(insn[i],"AND"); type=ALU; break;
6827           case 0x25: strcpy(insn[i],"OR"); type=ALU; break;
6828           case 0x26: strcpy(insn[i],"XOR"); type=ALU; break;
6829           case 0x27: strcpy(insn[i],"NOR"); type=ALU; break;
6830           case 0x2A: strcpy(insn[i],"SLT"); type=ALU; break;
6831           case 0x2B: strcpy(insn[i],"SLTU"); type=ALU; break;
6832           case 0x30: strcpy(insn[i],"TGE"); type=NI; break;
6833           case 0x31: strcpy(insn[i],"TGEU"); type=NI; break;
6834           case 0x32: strcpy(insn[i],"TLT"); type=NI; break;
6835           case 0x33: strcpy(insn[i],"TLTU"); type=NI; break;
6836           case 0x34: strcpy(insn[i],"TEQ"); type=NI; break;
6837           case 0x36: strcpy(insn[i],"TNE"); type=NI; break;
6838 #if 0
6839           case 0x14: strcpy(insn[i],"DSLLV"); type=SHIFT; break;
6840           case 0x16: strcpy(insn[i],"DSRLV"); type=SHIFT; break;
6841           case 0x17: strcpy(insn[i],"DSRAV"); type=SHIFT; break;
6842           case 0x1C: strcpy(insn[i],"DMULT"); type=MULTDIV; break;
6843           case 0x1D: strcpy(insn[i],"DMULTU"); type=MULTDIV; break;
6844           case 0x1E: strcpy(insn[i],"DDIV"); type=MULTDIV; break;
6845           case 0x1F: strcpy(insn[i],"DDIVU"); type=MULTDIV; break;
6846           case 0x2C: strcpy(insn[i],"DADD"); type=ALU; break;
6847           case 0x2D: strcpy(insn[i],"DADDU"); type=ALU; break;
6848           case 0x2E: strcpy(insn[i],"DSUB"); type=ALU; break;
6849           case 0x2F: strcpy(insn[i],"DSUBU"); type=ALU; break;
6850           case 0x38: strcpy(insn[i],"DSLL"); type=SHIFTIMM; break;
6851           case 0x3A: strcpy(insn[i],"DSRL"); type=SHIFTIMM; break;
6852           case 0x3B: strcpy(insn[i],"DSRA"); type=SHIFTIMM; break;
6853           case 0x3C: strcpy(insn[i],"DSLL32"); type=SHIFTIMM; break;
6854           case 0x3E: strcpy(insn[i],"DSRL32"); type=SHIFTIMM; break;
6855           case 0x3F: strcpy(insn[i],"DSRA32"); type=SHIFTIMM; break;
6856 #endif
6857         }
6858         break;
6859       case 0x01: strcpy(insn[i],"regimm"); type=NI;
6860         op2=(source[i]>>16)&0x1f;
6861         switch(op2)
6862         {
6863           case 0x00: strcpy(insn[i],"BLTZ"); type=SJUMP; break;
6864           case 0x01: strcpy(insn[i],"BGEZ"); type=SJUMP; break;
6865           case 0x02: strcpy(insn[i],"BLTZL"); type=SJUMP; break;
6866           case 0x03: strcpy(insn[i],"BGEZL"); type=SJUMP; break;
6867           case 0x08: strcpy(insn[i],"TGEI"); type=NI; break;
6868           case 0x09: strcpy(insn[i],"TGEIU"); type=NI; break;
6869           case 0x0A: strcpy(insn[i],"TLTI"); type=NI; break;
6870           case 0x0B: strcpy(insn[i],"TLTIU"); type=NI; break;
6871           case 0x0C: strcpy(insn[i],"TEQI"); type=NI; break;
6872           case 0x0E: strcpy(insn[i],"TNEI"); type=NI; break;
6873           case 0x10: strcpy(insn[i],"BLTZAL"); type=SJUMP; break;
6874           case 0x11: strcpy(insn[i],"BGEZAL"); type=SJUMP; break;
6875           case 0x12: strcpy(insn[i],"BLTZALL"); type=SJUMP; break;
6876           case 0x13: strcpy(insn[i],"BGEZALL"); type=SJUMP; break;
6877         }
6878         break;
6879       case 0x02: strcpy(insn[i],"J"); type=UJUMP; break;
6880       case 0x03: strcpy(insn[i],"JAL"); type=UJUMP; break;
6881       case 0x04: strcpy(insn[i],"BEQ"); type=CJUMP; break;
6882       case 0x05: strcpy(insn[i],"BNE"); type=CJUMP; break;
6883       case 0x06: strcpy(insn[i],"BLEZ"); type=CJUMP; break;
6884       case 0x07: strcpy(insn[i],"BGTZ"); type=CJUMP; break;
6885       case 0x08: strcpy(insn[i],"ADDI"); type=IMM16; break;
6886       case 0x09: strcpy(insn[i],"ADDIU"); type=IMM16; break;
6887       case 0x0A: strcpy(insn[i],"SLTI"); type=IMM16; break;
6888       case 0x0B: strcpy(insn[i],"SLTIU"); type=IMM16; break;
6889       case 0x0C: strcpy(insn[i],"ANDI"); type=IMM16; break;
6890       case 0x0D: strcpy(insn[i],"ORI"); type=IMM16; break;
6891       case 0x0E: strcpy(insn[i],"XORI"); type=IMM16; break;
6892       case 0x0F: strcpy(insn[i],"LUI"); type=IMM16; break;
6893       case 0x10: strcpy(insn[i],"cop0"); type=NI;
6894         op2=(source[i]>>21)&0x1f;
6895         switch(op2)
6896         {
6897           case 0x00: strcpy(insn[i],"MFC0"); type=COP0; break;
6898           case 0x02: strcpy(insn[i],"CFC0"); type=COP0; break;
6899           case 0x04: strcpy(insn[i],"MTC0"); type=COP0; break;
6900           case 0x06: strcpy(insn[i],"CTC0"); type=COP0; break;
6901           case 0x10: strcpy(insn[i],"RFE"); type=COP0; break;
6902         }
6903         break;
6904       case 0x11: strcpy(insn[i],"cop1"); type=COP1;
6905         op2=(source[i]>>21)&0x1f;
6906         break;
6907 #if 0
6908       case 0x14: strcpy(insn[i],"BEQL"); type=CJUMP; break;
6909       case 0x15: strcpy(insn[i],"BNEL"); type=CJUMP; break;
6910       case 0x16: strcpy(insn[i],"BLEZL"); type=CJUMP; break;
6911       case 0x17: strcpy(insn[i],"BGTZL"); type=CJUMP; break;
6912       case 0x18: strcpy(insn[i],"DADDI"); type=IMM16; break;
6913       case 0x19: strcpy(insn[i],"DADDIU"); type=IMM16; break;
6914       case 0x1A: strcpy(insn[i],"LDL"); type=LOADLR; break;
6915       case 0x1B: strcpy(insn[i],"LDR"); type=LOADLR; break;
6916 #endif
6917       case 0x20: strcpy(insn[i],"LB"); type=LOAD; break;
6918       case 0x21: strcpy(insn[i],"LH"); type=LOAD; break;
6919       case 0x22: strcpy(insn[i],"LWL"); type=LOADLR; break;
6920       case 0x23: strcpy(insn[i],"LW"); type=LOAD; break;
6921       case 0x24: strcpy(insn[i],"LBU"); type=LOAD; break;
6922       case 0x25: strcpy(insn[i],"LHU"); type=LOAD; break;
6923       case 0x26: strcpy(insn[i],"LWR"); type=LOADLR; break;
6924 #if 0
6925       case 0x27: strcpy(insn[i],"LWU"); type=LOAD; break;
6926 #endif
6927       case 0x28: strcpy(insn[i],"SB"); type=STORE; break;
6928       case 0x29: strcpy(insn[i],"SH"); type=STORE; break;
6929       case 0x2A: strcpy(insn[i],"SWL"); type=STORELR; break;
6930       case 0x2B: strcpy(insn[i],"SW"); type=STORE; break;
6931 #if 0
6932       case 0x2C: strcpy(insn[i],"SDL"); type=STORELR; break;
6933       case 0x2D: strcpy(insn[i],"SDR"); type=STORELR; break;
6934 #endif
6935       case 0x2E: strcpy(insn[i],"SWR"); type=STORELR; break;
6936       case 0x2F: strcpy(insn[i],"CACHE"); type=NOP; break;
6937       case 0x30: strcpy(insn[i],"LL"); type=NI; break;
6938       case 0x31: strcpy(insn[i],"LWC1"); type=C1LS; break;
6939 #if 0
6940       case 0x34: strcpy(insn[i],"LLD"); type=NI; break;
6941       case 0x35: strcpy(insn[i],"LDC1"); type=C1LS; break;
6942       case 0x37: strcpy(insn[i],"LD"); type=LOAD; break;
6943 #endif
6944       case 0x38: strcpy(insn[i],"SC"); type=NI; break;
6945       case 0x39: strcpy(insn[i],"SWC1"); type=C1LS; break;
6946 #if 0
6947       case 0x3C: strcpy(insn[i],"SCD"); type=NI; break;
6948       case 0x3D: strcpy(insn[i],"SDC1"); type=C1LS; break;
6949       case 0x3F: strcpy(insn[i],"SD"); type=STORE; break;
6950 #endif
6951       case 0x12: strcpy(insn[i],"COP2"); type=NI;
6952         op2=(source[i]>>21)&0x1f;
6953         //if (op2 & 0x10)
6954         if (source[i]&0x3f) { // use this hack to support old savestates with patched gte insns
6955           if (gte_handlers[source[i]&0x3f]!=NULL) {
6956             if (gte_regnames[source[i]&0x3f]!=NULL)
6957               strcpy(insn[i],gte_regnames[source[i]&0x3f]);
6958             else
6959               snprintf(insn[i], sizeof(insn[i]), "COP2 %x", source[i]&0x3f);
6960             type=C2OP;
6961           }
6962         }
6963         else switch(op2)
6964         {
6965           case 0x00: strcpy(insn[i],"MFC2"); type=COP2; break;
6966           case 0x02: strcpy(insn[i],"CFC2"); type=COP2; break;
6967           case 0x04: strcpy(insn[i],"MTC2"); type=COP2; break;
6968           case 0x06: strcpy(insn[i],"CTC2"); type=COP2; break;
6969         }
6970         break;
6971       case 0x32: strcpy(insn[i],"LWC2"); type=C2LS; break;
6972       case 0x3A: strcpy(insn[i],"SWC2"); type=C2LS; break;
6973       case 0x3B: strcpy(insn[i],"HLECALL"); type=HLECALL; break;
6974       default: strcpy(insn[i],"???"); type=NI;
6975         SysPrintf("NI %08x @%08x (%08x)\n", source[i], addr + i*4, addr);
6976         break;
6977     }
6978     itype[i]=type;
6979     opcode2[i]=op2;
6980     /* Get registers/immediates */
6981     lt1[i]=0;
6982     dep1[i]=0;
6983     dep2[i]=0;
6984     gte_rs[i]=gte_rt[i]=0;
6985     switch(type) {
6986       case LOAD:
6987         rs1[i]=(source[i]>>21)&0x1f;
6988         rs2[i]=0;
6989         rt1[i]=(source[i]>>16)&0x1f;
6990         rt2[i]=0;
6991         imm[i]=(short)source[i];
6992         break;
6993       case STORE:
6994       case STORELR:
6995         rs1[i]=(source[i]>>21)&0x1f;
6996         rs2[i]=(source[i]>>16)&0x1f;
6997         rt1[i]=0;
6998         rt2[i]=0;
6999         imm[i]=(short)source[i];
7000         break;
7001       case LOADLR:
7002         // LWL/LWR only load part of the register,
7003         // therefore the target register must be treated as a source too
7004         rs1[i]=(source[i]>>21)&0x1f;
7005         rs2[i]=(source[i]>>16)&0x1f;
7006         rt1[i]=(source[i]>>16)&0x1f;
7007         rt2[i]=0;
7008         imm[i]=(short)source[i];
7009         if(op==0x26) dep1[i]=rt1[i]; // LWR
7010         break;
7011       case IMM16:
7012         if (op==0x0f) rs1[i]=0; // LUI instruction has no source register
7013         else rs1[i]=(source[i]>>21)&0x1f;
7014         rs2[i]=0;
7015         rt1[i]=(source[i]>>16)&0x1f;
7016         rt2[i]=0;
7017         if(op>=0x0c&&op<=0x0e) { // ANDI/ORI/XORI
7018           imm[i]=(unsigned short)source[i];
7019         }else{
7020           imm[i]=(short)source[i];
7021         }
7022         if(op==0x0d||op==0x0e) dep1[i]=rs1[i]; // ORI/XORI
7023         break;
7024       case UJUMP:
7025         rs1[i]=0;
7026         rs2[i]=0;
7027         rt1[i]=0;
7028         rt2[i]=0;
7029         // The JAL instruction writes to r31.
7030         if (op&1) {
7031           rt1[i]=31;
7032         }
7033         rs2[i]=CCREG;
7034         break;
7035       case RJUMP:
7036         rs1[i]=(source[i]>>21)&0x1f;
7037         rs2[i]=0;
7038         rt1[i]=0;
7039         rt2[i]=0;
7040         // The JALR instruction writes to rd.
7041         if (op2&1) {
7042           rt1[i]=(source[i]>>11)&0x1f;
7043         }
7044         rs2[i]=CCREG;
7045         break;
7046       case CJUMP:
7047         rs1[i]=(source[i]>>21)&0x1f;
7048         rs2[i]=(source[i]>>16)&0x1f;
7049         rt1[i]=0;
7050         rt2[i]=0;
7051         if(op&2) { // BGTZ/BLEZ
7052           rs2[i]=0;
7053         }
7054         likely[i]=op>>4;
7055         break;
7056       case SJUMP:
7057         rs1[i]=(source[i]>>21)&0x1f;
7058         rs2[i]=CCREG;
7059         rt1[i]=0;
7060         rt2[i]=0;
7061         if(op2&0x10) { // BxxAL
7062           rt1[i]=31;
7063           // NOTE: If the branch is not taken, r31 is still overwritten
7064         }
7065         likely[i]=(op2&2)>>1;
7066         break;
7067       case ALU:
7068         rs1[i]=(source[i]>>21)&0x1f; // source
7069         rs2[i]=(source[i]>>16)&0x1f; // subtract amount
7070         rt1[i]=(source[i]>>11)&0x1f; // destination
7071         rt2[i]=0;
7072         if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
7073           dep1[i]=rs1[i];dep2[i]=rs2[i];
7074         }
7075         else if(op2>=0x2c&&op2<=0x2f) { // DADD/DSUB
7076           dep1[i]=rs1[i];dep2[i]=rs2[i];
7077         }
7078         break;
7079       case MULTDIV:
7080         rs1[i]=(source[i]>>21)&0x1f; // source
7081         rs2[i]=(source[i]>>16)&0x1f; // divisor
7082         rt1[i]=HIREG;
7083         rt2[i]=LOREG;
7084         break;
7085       case MOV:
7086         rs1[i]=0;
7087         rs2[i]=0;
7088         rt1[i]=0;
7089         rt2[i]=0;
7090         if(op2==0x10) rs1[i]=HIREG; // MFHI
7091         if(op2==0x11) rt1[i]=HIREG; // MTHI
7092         if(op2==0x12) rs1[i]=LOREG; // MFLO
7093         if(op2==0x13) rt1[i]=LOREG; // MTLO
7094         if((op2&0x1d)==0x10) rt1[i]=(source[i]>>11)&0x1f; // MFxx
7095         if((op2&0x1d)==0x11) rs1[i]=(source[i]>>21)&0x1f; // MTxx
7096         dep1[i]=rs1[i];
7097         break;
7098       case SHIFT:
7099         rs1[i]=(source[i]>>16)&0x1f; // target of shift
7100         rs2[i]=(source[i]>>21)&0x1f; // shift amount
7101         rt1[i]=(source[i]>>11)&0x1f; // destination
7102         rt2[i]=0;
7103         break;
7104       case SHIFTIMM:
7105         rs1[i]=(source[i]>>16)&0x1f;
7106         rs2[i]=0;
7107         rt1[i]=(source[i]>>11)&0x1f;
7108         rt2[i]=0;
7109         imm[i]=(source[i]>>6)&0x1f;
7110         // DSxx32 instructions
7111         if(op2>=0x3c) imm[i]|=0x20;
7112         break;
7113       case COP0:
7114         rs1[i]=0;
7115         rs2[i]=0;
7116         rt1[i]=0;
7117         rt2[i]=0;
7118         if(op2==0||op2==2) rt1[i]=(source[i]>>16)&0x1F; // MFC0/CFC0
7119         if(op2==4||op2==6) rs1[i]=(source[i]>>16)&0x1F; // MTC0/CTC0
7120         if(op2==4&&((source[i]>>11)&0x1f)==12) rt2[i]=CSREG; // Status
7121         if(op2==16) if((source[i]&0x3f)==0x18) rs2[i]=CCREG; // ERET
7122         break;
7123       case COP1:
7124         rs1[i]=0;
7125         rs2[i]=0;
7126         rt1[i]=0;
7127         rt2[i]=0;
7128         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC1/DMFC1/CFC1
7129         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC1/DMTC1/CTC1
7130         rs2[i]=CSREG;
7131         break;
7132       case COP2:
7133         rs1[i]=0;
7134         rs2[i]=0;
7135         rt1[i]=0;
7136         rt2[i]=0;
7137         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC2/CFC2
7138         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC2/CTC2
7139         rs2[i]=CSREG;
7140         int gr=(source[i]>>11)&0x1F;
7141         switch(op2)
7142         {
7143           case 0x00: gte_rs[i]=1ll<<gr; break; // MFC2
7144           case 0x04: gte_rt[i]=1ll<<gr; break; // MTC2
7145           case 0x02: gte_rs[i]=1ll<<(gr+32); break; // CFC2
7146           case 0x06: gte_rt[i]=1ll<<(gr+32); break; // CTC2
7147         }
7148         break;
7149       case C1LS:
7150         rs1[i]=(source[i]>>21)&0x1F;
7151         rs2[i]=CSREG;
7152         rt1[i]=0;
7153         rt2[i]=0;
7154         imm[i]=(short)source[i];
7155         break;
7156       case C2LS:
7157         rs1[i]=(source[i]>>21)&0x1F;
7158         rs2[i]=0;
7159         rt1[i]=0;
7160         rt2[i]=0;
7161         imm[i]=(short)source[i];
7162         if(op==0x32) gte_rt[i]=1ll<<((source[i]>>16)&0x1F); // LWC2
7163         else gte_rs[i]=1ll<<((source[i]>>16)&0x1F); // SWC2
7164         break;
7165       case C2OP:
7166         rs1[i]=0;
7167         rs2[i]=0;
7168         rt1[i]=0;
7169         rt2[i]=0;
7170         gte_rs[i]=gte_reg_reads[source[i]&0x3f];
7171         gte_rt[i]=gte_reg_writes[source[i]&0x3f];
7172         gte_rt[i]|=1ll<<63; // every op changes flags
7173         if((source[i]&0x3f)==GTE_MVMVA) {
7174           int v = (source[i] >> 15) & 3;
7175           gte_rs[i]&=~0xe3fll;
7176           if(v==3) gte_rs[i]|=0xe00ll;
7177           else gte_rs[i]|=3ll<<(v*2);
7178         }
7179         break;
7180       case SYSCALL:
7181       case HLECALL:
7182       case INTCALL:
7183         rs1[i]=CCREG;
7184         rs2[i]=0;
7185         rt1[i]=0;
7186         rt2[i]=0;
7187         break;
7188       default:
7189         rs1[i]=0;
7190         rs2[i]=0;
7191         rt1[i]=0;
7192         rt2[i]=0;
7193     }
7194     /* Calculate branch target addresses */
7195     if(type==UJUMP)
7196       ba[i]=((start+i*4+4)&0xF0000000)|(((unsigned int)source[i]<<6)>>4);
7197     else if(type==CJUMP&&rs1[i]==rs2[i]&&(op&1))
7198       ba[i]=start+i*4+8; // Ignore never taken branch
7199     else if(type==SJUMP&&rs1[i]==0&&!(op2&1))
7200       ba[i]=start+i*4+8; // Ignore never taken branch
7201     else if(type==CJUMP||type==SJUMP)
7202       ba[i]=start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14);
7203     else ba[i]=-1;
7204     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
7205       int do_in_intrp=0;
7206       // branch in delay slot?
7207       if(type==RJUMP||type==UJUMP||type==CJUMP||type==SJUMP) {
7208         // don't handle first branch and call interpreter if it's hit
7209         SysPrintf("branch in delay slot @%08x (%08x)\n", addr + i*4, addr);
7210         do_in_intrp=1;
7211       }
7212       // basic load delay detection
7213       else if((type==LOAD||type==LOADLR||type==COP0||type==COP2||type==C2LS)&&rt1[i]!=0) {
7214         int t=(ba[i-1]-start)/4;
7215         if(0 <= t && t < i &&(rt1[i]==rs1[t]||rt1[i]==rs2[t])&&itype[t]!=CJUMP&&itype[t]!=SJUMP) {
7216           // jump target wants DS result - potential load delay effect
7217           SysPrintf("load delay @%08x (%08x)\n", addr + i*4, addr);
7218           do_in_intrp=1;
7219           bt[t+1]=1; // expected return from interpreter
7220         }
7221         else if(i>=2&&rt1[i-2]==2&&rt1[i]==2&&rs1[i]!=2&&rs2[i]!=2&&rs1[i-1]!=2&&rs2[i-1]!=2&&
7222               !(i>=3&&(itype[i-3]==RJUMP||itype[i-3]==UJUMP||itype[i-3]==CJUMP||itype[i-3]==SJUMP))) {
7223           // v0 overwrite like this is a sign of trouble, bail out
7224           SysPrintf("v0 overwrite @%08x (%08x)\n", addr + i*4, addr);
7225           do_in_intrp=1;
7226         }
7227       }
7228       if(do_in_intrp) {
7229         rs1[i-1]=CCREG;
7230         rs2[i-1]=rt1[i-1]=rt2[i-1]=0;
7231         ba[i-1]=-1;
7232         itype[i-1]=INTCALL;
7233         done=2;
7234         i--; // don't compile the DS
7235       }
7236     }
7237     /* Is this the end of the block? */
7238     if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)) {
7239       if(rt1[i-1]==0) { // Continue past subroutine call (JAL)
7240         done=2;
7241       }
7242       else {
7243         if(stop_after_jal) done=1;
7244         // Stop on BREAK
7245         if((source[i+1]&0xfc00003f)==0x0d) done=1;
7246       }
7247       // Don't recompile stuff that's already compiled
7248       if(check_addr(start+i*4+4)) done=1;
7249       // Don't get too close to the limit
7250       if(i>MAXBLOCK/2) done=1;
7251     }
7252     if(itype[i]==SYSCALL&&stop_after_jal) done=1;
7253     if(itype[i]==HLECALL||itype[i]==INTCALL) done=2;
7254     if(done==2) {
7255       // Does the block continue due to a branch?
7256       for(j=i-1;j>=0;j--)
7257       {
7258         if(ba[j]==start+i*4) done=j=0; // Branch into delay slot
7259         if(ba[j]==start+i*4+4) done=j=0;
7260         if(ba[j]==start+i*4+8) done=j=0;
7261       }
7262     }
7263     //assert(i<MAXBLOCK-1);
7264     if(start+i*4==pagelimit-4) done=1;
7265     assert(start+i*4<pagelimit);
7266     if (i==MAXBLOCK-1) done=1;
7267     // Stop if we're compiling junk
7268     if(itype[i]==NI&&opcode[i]==0x11) {
7269       done=stop_after_jal=1;
7270       SysPrintf("Disabled speculative precompilation\n");
7271     }
7272   }
7273   slen=i;
7274   if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==RJUMP) {
7275     if(start+i*4==pagelimit) {
7276       itype[i-1]=SPAN;
7277     }
7278   }
7279   assert(slen>0);
7280
7281   /* Pass 2 - Register dependencies and branch targets */
7282
7283   unneeded_registers(0,slen-1,0);
7284
7285   /* Pass 3 - Register allocation */
7286
7287   struct regstat current; // Current register allocations/status
7288   current.dirty=0;
7289   current.u=unneeded_reg[0];
7290   clear_all_regs(current.regmap);
7291   alloc_reg(&current,0,CCREG);
7292   dirty_reg(&current,CCREG);
7293   current.isconst=0;
7294   current.wasconst=0;
7295   current.waswritten=0;
7296   int ds=0;
7297   int cc=0;
7298   int hr=-1;
7299
7300   if((u_int)addr&1) {
7301     // First instruction is delay slot
7302     cc=-1;
7303     bt[1]=1;
7304     ds=1;
7305     unneeded_reg[0]=1;
7306     current.regmap[HOST_BTREG]=BTREG;
7307   }
7308
7309   for(i=0;i<slen;i++)
7310   {
7311     if(bt[i])
7312     {
7313       int hr;
7314       for(hr=0;hr<HOST_REGS;hr++)
7315       {
7316         // Is this really necessary?
7317         if(current.regmap[hr]==0) current.regmap[hr]=-1;
7318       }
7319       current.isconst=0;
7320       current.waswritten=0;
7321     }
7322
7323     memcpy(regmap_pre[i],current.regmap,sizeof(current.regmap));
7324     regs[i].wasconst=current.isconst;
7325     regs[i].wasdirty=current.dirty;
7326     regs[i].loadedconst=0;
7327     if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP) {
7328       if(i+1<slen) {
7329         current.u=unneeded_reg[i+1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7330         current.u|=1;
7331       } else {
7332         current.u=1;
7333       }
7334     } else {
7335       if(i+1<slen) {
7336         current.u=branch_unneeded_reg[i]&~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
7337         current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7338         current.u|=1;
7339       } else { SysPrintf("oops, branch at end of block with no delay slot\n");abort(); }
7340     }
7341     is_ds[i]=ds;
7342     if(ds) {
7343       ds=0; // Skip delay slot, already allocated as part of branch
7344       // ...but we need to alloc it in case something jumps here
7345       if(i+1<slen) {
7346         current.u=branch_unneeded_reg[i-1]&unneeded_reg[i+1];
7347       }else{
7348         current.u=branch_unneeded_reg[i-1];
7349       }
7350       current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7351       current.u|=1;
7352       struct regstat temp;
7353       memcpy(&temp,&current,sizeof(current));
7354       temp.wasdirty=temp.dirty;
7355       // TODO: Take into account unconditional branches, as below
7356       delayslot_alloc(&temp,i);
7357       memcpy(regs[i].regmap,temp.regmap,sizeof(temp.regmap));
7358       regs[i].wasdirty=temp.wasdirty;
7359       regs[i].dirty=temp.dirty;
7360       regs[i].isconst=0;
7361       regs[i].wasconst=0;
7362       current.isconst=0;
7363       // Create entry (branch target) regmap
7364       for(hr=0;hr<HOST_REGS;hr++)
7365       {
7366         int r=temp.regmap[hr];
7367         if(r>=0) {
7368           if(r!=regmap_pre[i][hr]) {
7369             regs[i].regmap_entry[hr]=-1;
7370           }
7371           else
7372           {
7373               assert(r < 64);
7374               if((current.u>>r)&1) {
7375                 regs[i].regmap_entry[hr]=-1;
7376                 regs[i].regmap[hr]=-1;
7377                 //Don't clear regs in the delay slot as the branch might need them
7378                 //current.regmap[hr]=-1;
7379               }else
7380                 regs[i].regmap_entry[hr]=r;
7381           }
7382         } else {
7383           // First instruction expects CCREG to be allocated
7384           if(i==0&&hr==HOST_CCREG)
7385             regs[i].regmap_entry[hr]=CCREG;
7386           else
7387             regs[i].regmap_entry[hr]=-1;
7388         }
7389       }
7390     }
7391     else { // Not delay slot
7392       switch(itype[i]) {
7393         case UJUMP:
7394           //current.isconst=0; // DEBUG
7395           //current.wasconst=0; // DEBUG
7396           //regs[i].wasconst=0; // DEBUG
7397           clear_const(&current,rt1[i]);
7398           alloc_cc(&current,i);
7399           dirty_reg(&current,CCREG);
7400           if (rt1[i]==31) {
7401             alloc_reg(&current,i,31);
7402             dirty_reg(&current,31);
7403             //assert(rs1[i+1]!=31&&rs2[i+1]!=31);
7404             //assert(rt1[i+1]!=rt1[i]);
7405             #ifdef REG_PREFETCH
7406             alloc_reg(&current,i,PTEMP);
7407             #endif
7408           }
7409           ooo[i]=1;
7410           delayslot_alloc(&current,i+1);
7411           //current.isconst=0; // DEBUG
7412           ds=1;
7413           //printf("i=%d, isconst=%x\n",i,current.isconst);
7414           break;
7415         case RJUMP:
7416           //current.isconst=0;
7417           //current.wasconst=0;
7418           //regs[i].wasconst=0;
7419           clear_const(&current,rs1[i]);
7420           clear_const(&current,rt1[i]);
7421           alloc_cc(&current,i);
7422           dirty_reg(&current,CCREG);
7423           if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
7424             alloc_reg(&current,i,rs1[i]);
7425             if (rt1[i]!=0) {
7426               alloc_reg(&current,i,rt1[i]);
7427               dirty_reg(&current,rt1[i]);
7428               assert(rs1[i+1]!=rt1[i]&&rs2[i+1]!=rt1[i]);
7429               assert(rt1[i+1]!=rt1[i]);
7430               #ifdef REG_PREFETCH
7431               alloc_reg(&current,i,PTEMP);
7432               #endif
7433             }
7434             #ifdef USE_MINI_HT
7435             if(rs1[i]==31) { // JALR
7436               alloc_reg(&current,i,RHASH);
7437               alloc_reg(&current,i,RHTBL);
7438             }
7439             #endif
7440             delayslot_alloc(&current,i+1);
7441           } else {
7442             // The delay slot overwrites our source register,
7443             // allocate a temporary register to hold the old value.
7444             current.isconst=0;
7445             current.wasconst=0;
7446             regs[i].wasconst=0;
7447             delayslot_alloc(&current,i+1);
7448             current.isconst=0;
7449             alloc_reg(&current,i,RTEMP);
7450           }
7451           //current.isconst=0; // DEBUG
7452           ooo[i]=1;
7453           ds=1;
7454           break;
7455         case CJUMP:
7456           //current.isconst=0;
7457           //current.wasconst=0;
7458           //regs[i].wasconst=0;
7459           clear_const(&current,rs1[i]);
7460           clear_const(&current,rs2[i]);
7461           if((opcode[i]&0x3E)==4) // BEQ/BNE
7462           {
7463             alloc_cc(&current,i);
7464             dirty_reg(&current,CCREG);
7465             if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7466             if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7467             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]))||
7468                (rs2[i]&&(rs2[i]==rt1[i+1]||rs2[i]==rt2[i+1]))) {
7469               // The delay slot overwrites one of our conditions.
7470               // Allocate the branch condition registers instead.
7471               current.isconst=0;
7472               current.wasconst=0;
7473               regs[i].wasconst=0;
7474               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7475               if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7476             }
7477             else
7478             {
7479               ooo[i]=1;
7480               delayslot_alloc(&current,i+1);
7481             }
7482           }
7483           else
7484           if((opcode[i]&0x3E)==6) // BLEZ/BGTZ
7485           {
7486             alloc_cc(&current,i);
7487             dirty_reg(&current,CCREG);
7488             alloc_reg(&current,i,rs1[i]);
7489             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
7490               // The delay slot overwrites one of our conditions.
7491               // Allocate the branch condition registers instead.
7492               current.isconst=0;
7493               current.wasconst=0;
7494               regs[i].wasconst=0;
7495               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7496             }
7497             else
7498             {
7499               ooo[i]=1;
7500               delayslot_alloc(&current,i+1);
7501             }
7502           }
7503           else
7504           // Don't alloc the delay slot yet because we might not execute it
7505           if((opcode[i]&0x3E)==0x14) // BEQL/BNEL
7506           {
7507             current.isconst=0;
7508             current.wasconst=0;
7509             regs[i].wasconst=0;
7510             alloc_cc(&current,i);
7511             dirty_reg(&current,CCREG);
7512             alloc_reg(&current,i,rs1[i]);
7513             alloc_reg(&current,i,rs2[i]);
7514           }
7515           else
7516           if((opcode[i]&0x3E)==0x16) // BLEZL/BGTZL
7517           {
7518             current.isconst=0;
7519             current.wasconst=0;
7520             regs[i].wasconst=0;
7521             alloc_cc(&current,i);
7522             dirty_reg(&current,CCREG);
7523             alloc_reg(&current,i,rs1[i]);
7524           }
7525           ds=1;
7526           //current.isconst=0;
7527           break;
7528         case SJUMP:
7529           //current.isconst=0;
7530           //current.wasconst=0;
7531           //regs[i].wasconst=0;
7532           clear_const(&current,rs1[i]);
7533           clear_const(&current,rt1[i]);
7534           //if((opcode2[i]&0x1E)==0x0) // BLTZ/BGEZ
7535           if((opcode2[i]&0x0E)==0x0) // BLTZ/BGEZ
7536           {
7537             alloc_cc(&current,i);
7538             dirty_reg(&current,CCREG);
7539             alloc_reg(&current,i,rs1[i]);
7540             if (rt1[i]==31) { // BLTZAL/BGEZAL
7541               alloc_reg(&current,i,31);
7542               dirty_reg(&current,31);
7543               //#ifdef REG_PREFETCH
7544               //alloc_reg(&current,i,PTEMP);
7545               //#endif
7546             }
7547             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) // The delay slot overwrites the branch condition.
7548                ||(rt1[i]==31&&(rs1[i+1]==31||rs2[i+1]==31||rt1[i+1]==31||rt2[i+1]==31))) { // DS touches $ra
7549               // Allocate the branch condition registers instead.
7550               current.isconst=0;
7551               current.wasconst=0;
7552               regs[i].wasconst=0;
7553               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7554             }
7555             else
7556             {
7557               ooo[i]=1;
7558               delayslot_alloc(&current,i+1);
7559             }
7560           }
7561           else
7562           // Don't alloc the delay slot yet because we might not execute it
7563           if((opcode2[i]&0x1E)==0x2) // BLTZL/BGEZL
7564           {
7565             current.isconst=0;
7566             current.wasconst=0;
7567             regs[i].wasconst=0;
7568             alloc_cc(&current,i);
7569             dirty_reg(&current,CCREG);
7570             alloc_reg(&current,i,rs1[i]);
7571           }
7572           ds=1;
7573           //current.isconst=0;
7574           break;
7575         case IMM16:
7576           imm16_alloc(&current,i);
7577           break;
7578         case LOAD:
7579         case LOADLR:
7580           load_alloc(&current,i);
7581           break;
7582         case STORE:
7583         case STORELR:
7584           store_alloc(&current,i);
7585           break;
7586         case ALU:
7587           alu_alloc(&current,i);
7588           break;
7589         case SHIFT:
7590           shift_alloc(&current,i);
7591           break;
7592         case MULTDIV:
7593           multdiv_alloc(&current,i);
7594           break;
7595         case SHIFTIMM:
7596           shiftimm_alloc(&current,i);
7597           break;
7598         case MOV:
7599           mov_alloc(&current,i);
7600           break;
7601         case COP0:
7602           cop0_alloc(&current,i);
7603           break;
7604         case COP1:
7605         case COP2:
7606           cop12_alloc(&current,i);
7607           break;
7608         case C1LS:
7609           c1ls_alloc(&current,i);
7610           break;
7611         case C2LS:
7612           c2ls_alloc(&current,i);
7613           break;
7614         case C2OP:
7615           c2op_alloc(&current,i);
7616           break;
7617         case SYSCALL:
7618         case HLECALL:
7619         case INTCALL:
7620           syscall_alloc(&current,i);
7621           break;
7622         case SPAN:
7623           pagespan_alloc(&current,i);
7624           break;
7625       }
7626
7627       // Create entry (branch target) regmap
7628       for(hr=0;hr<HOST_REGS;hr++)
7629       {
7630         int r,or;
7631         r=current.regmap[hr];
7632         if(r>=0) {
7633           if(r!=regmap_pre[i][hr]) {
7634             // TODO: delay slot (?)
7635             or=get_reg(regmap_pre[i],r); // Get old mapping for this register
7636             if(or<0||(r&63)>=TEMPREG){
7637               regs[i].regmap_entry[hr]=-1;
7638             }
7639             else
7640             {
7641               // Just move it to a different register
7642               regs[i].regmap_entry[hr]=r;
7643               // If it was dirty before, it's still dirty
7644               if((regs[i].wasdirty>>or)&1) dirty_reg(&current,r&63);
7645             }
7646           }
7647           else
7648           {
7649             // Unneeded
7650             if(r==0){
7651               regs[i].regmap_entry[hr]=0;
7652             }
7653             else
7654             {
7655               assert(r<64);
7656               if((current.u>>r)&1) {
7657                 regs[i].regmap_entry[hr]=-1;
7658                 //regs[i].regmap[hr]=-1;
7659                 current.regmap[hr]=-1;
7660               }else
7661                 regs[i].regmap_entry[hr]=r;
7662             }
7663           }
7664         } else {
7665           // Branches expect CCREG to be allocated at the target
7666           if(regmap_pre[i][hr]==CCREG)
7667             regs[i].regmap_entry[hr]=CCREG;
7668           else
7669             regs[i].regmap_entry[hr]=-1;
7670         }
7671       }
7672       memcpy(regs[i].regmap,current.regmap,sizeof(current.regmap));
7673     }
7674
7675     if(i>0&&(itype[i-1]==STORE||itype[i-1]==STORELR||(itype[i-1]==C2LS&&opcode[i-1]==0x3a))&&(u_int)imm[i-1]<0x800)
7676       current.waswritten|=1<<rs1[i-1];
7677     current.waswritten&=~(1<<rt1[i]);
7678     current.waswritten&=~(1<<rt2[i]);
7679     if((itype[i]==STORE||itype[i]==STORELR||(itype[i]==C2LS&&opcode[i]==0x3a))&&(u_int)imm[i]>=0x800)
7680       current.waswritten&=~(1<<rs1[i]);
7681
7682     /* Branch post-alloc */
7683     if(i>0)
7684     {
7685       current.wasdirty=current.dirty;
7686       switch(itype[i-1]) {
7687         case UJUMP:
7688           memcpy(&branch_regs[i-1],&current,sizeof(current));
7689           branch_regs[i-1].isconst=0;
7690           branch_regs[i-1].wasconst=0;
7691           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7692           alloc_cc(&branch_regs[i-1],i-1);
7693           dirty_reg(&branch_regs[i-1],CCREG);
7694           if(rt1[i-1]==31) { // JAL
7695             alloc_reg(&branch_regs[i-1],i-1,31);
7696             dirty_reg(&branch_regs[i-1],31);
7697           }
7698           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7699           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7700           break;
7701         case RJUMP:
7702           memcpy(&branch_regs[i-1],&current,sizeof(current));
7703           branch_regs[i-1].isconst=0;
7704           branch_regs[i-1].wasconst=0;
7705           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7706           alloc_cc(&branch_regs[i-1],i-1);
7707           dirty_reg(&branch_regs[i-1],CCREG);
7708           alloc_reg(&branch_regs[i-1],i-1,rs1[i-1]);
7709           if(rt1[i-1]!=0) { // JALR
7710             alloc_reg(&branch_regs[i-1],i-1,rt1[i-1]);
7711             dirty_reg(&branch_regs[i-1],rt1[i-1]);
7712           }
7713           #ifdef USE_MINI_HT
7714           if(rs1[i-1]==31) { // JALR
7715             alloc_reg(&branch_regs[i-1],i-1,RHASH);
7716             alloc_reg(&branch_regs[i-1],i-1,RHTBL);
7717           }
7718           #endif
7719           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7720           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7721           break;
7722         case CJUMP:
7723           if((opcode[i-1]&0x3E)==4) // BEQ/BNE
7724           {
7725             alloc_cc(&current,i-1);
7726             dirty_reg(&current,CCREG);
7727             if((rs1[i-1]&&(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]))||
7728                (rs2[i-1]&&(rs2[i-1]==rt1[i]||rs2[i-1]==rt2[i]))) {
7729               // The delay slot overwrote one of our conditions
7730               // Delay slot goes after the test (in order)
7731               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7732               current.u|=1;
7733               delayslot_alloc(&current,i);
7734               current.isconst=0;
7735             }
7736             else
7737             {
7738               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7739               // Alloc the branch condition registers
7740               if(rs1[i-1]) alloc_reg(&current,i-1,rs1[i-1]);
7741               if(rs2[i-1]) alloc_reg(&current,i-1,rs2[i-1]);
7742             }
7743             memcpy(&branch_regs[i-1],&current,sizeof(current));
7744             branch_regs[i-1].isconst=0;
7745             branch_regs[i-1].wasconst=0;
7746             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7747             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7748           }
7749           else
7750           if((opcode[i-1]&0x3E)==6) // BLEZ/BGTZ
7751           {
7752             alloc_cc(&current,i-1);
7753             dirty_reg(&current,CCREG);
7754             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7755               // The delay slot overwrote the branch condition
7756               // Delay slot goes after the test (in order)
7757               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7758               current.u|=1;
7759               delayslot_alloc(&current,i);
7760               current.isconst=0;
7761             }
7762             else
7763             {
7764               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7765               // Alloc the branch condition register
7766               alloc_reg(&current,i-1,rs1[i-1]);
7767             }
7768             memcpy(&branch_regs[i-1],&current,sizeof(current));
7769             branch_regs[i-1].isconst=0;
7770             branch_regs[i-1].wasconst=0;
7771             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7772             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7773           }
7774           else
7775           // Alloc the delay slot in case the branch is taken
7776           if((opcode[i-1]&0x3E)==0x14) // BEQL/BNEL
7777           {
7778             memcpy(&branch_regs[i-1],&current,sizeof(current));
7779             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7780             alloc_cc(&branch_regs[i-1],i);
7781             dirty_reg(&branch_regs[i-1],CCREG);
7782             delayslot_alloc(&branch_regs[i-1],i);
7783             branch_regs[i-1].isconst=0;
7784             alloc_reg(&current,i,CCREG); // Not taken path
7785             dirty_reg(&current,CCREG);
7786             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7787           }
7788           else
7789           if((opcode[i-1]&0x3E)==0x16) // BLEZL/BGTZL
7790           {
7791             memcpy(&branch_regs[i-1],&current,sizeof(current));
7792             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7793             alloc_cc(&branch_regs[i-1],i);
7794             dirty_reg(&branch_regs[i-1],CCREG);
7795             delayslot_alloc(&branch_regs[i-1],i);
7796             branch_regs[i-1].isconst=0;
7797             alloc_reg(&current,i,CCREG); // Not taken path
7798             dirty_reg(&current,CCREG);
7799             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7800           }
7801           break;
7802         case SJUMP:
7803           //if((opcode2[i-1]&0x1E)==0) // BLTZ/BGEZ
7804           if((opcode2[i-1]&0x0E)==0) // BLTZ/BGEZ
7805           {
7806             alloc_cc(&current,i-1);
7807             dirty_reg(&current,CCREG);
7808             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7809               // The delay slot overwrote the branch condition
7810               // Delay slot goes after the test (in order)
7811               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7812               current.u|=1;
7813               delayslot_alloc(&current,i);
7814               current.isconst=0;
7815             }
7816             else
7817             {
7818               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7819               // Alloc the branch condition register
7820               alloc_reg(&current,i-1,rs1[i-1]);
7821             }
7822             memcpy(&branch_regs[i-1],&current,sizeof(current));
7823             branch_regs[i-1].isconst=0;
7824             branch_regs[i-1].wasconst=0;
7825             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7826             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7827           }
7828           else
7829           // Alloc the delay slot in case the branch is taken
7830           if((opcode2[i-1]&0x1E)==2) // BLTZL/BGEZL
7831           {
7832             memcpy(&branch_regs[i-1],&current,sizeof(current));
7833             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7834             alloc_cc(&branch_regs[i-1],i);
7835             dirty_reg(&branch_regs[i-1],CCREG);
7836             delayslot_alloc(&branch_regs[i-1],i);
7837             branch_regs[i-1].isconst=0;
7838             alloc_reg(&current,i,CCREG); // Not taken path
7839             dirty_reg(&current,CCREG);
7840             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7841           }
7842           // FIXME: BLTZAL/BGEZAL
7843           if(opcode2[i-1]&0x10) { // BxxZAL
7844             alloc_reg(&branch_regs[i-1],i-1,31);
7845             dirty_reg(&branch_regs[i-1],31);
7846           }
7847           break;
7848       }
7849
7850       if(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)
7851       {
7852         if(rt1[i-1]==31) // JAL/JALR
7853         {
7854           // Subroutine call will return here, don't alloc any registers
7855           current.dirty=0;
7856           clear_all_regs(current.regmap);
7857           alloc_reg(&current,i,CCREG);
7858           dirty_reg(&current,CCREG);
7859         }
7860         else if(i+1<slen)
7861         {
7862           // Internal branch will jump here, match registers to caller
7863           current.dirty=0;
7864           clear_all_regs(current.regmap);
7865           alloc_reg(&current,i,CCREG);
7866           dirty_reg(&current,CCREG);
7867           for(j=i-1;j>=0;j--)
7868           {
7869             if(ba[j]==start+i*4+4) {
7870               memcpy(current.regmap,branch_regs[j].regmap,sizeof(current.regmap));
7871               current.dirty=branch_regs[j].dirty;
7872               break;
7873             }
7874           }
7875           while(j>=0) {
7876             if(ba[j]==start+i*4+4) {
7877               for(hr=0;hr<HOST_REGS;hr++) {
7878                 if(current.regmap[hr]!=branch_regs[j].regmap[hr]) {
7879                   current.regmap[hr]=-1;
7880                 }
7881                 current.dirty&=branch_regs[j].dirty;
7882               }
7883             }
7884             j--;
7885           }
7886         }
7887       }
7888     }
7889
7890     // Count cycles in between branches
7891     ccadj[i]=cc;
7892     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i]==SYSCALL||itype[i]==HLECALL))
7893     {
7894       cc=0;
7895     }
7896 #if !defined(DRC_DBG)
7897     else if(itype[i]==C2OP&&gte_cycletab[source[i]&0x3f]>2)
7898     {
7899       // GTE runs in parallel until accessed, divide by 2 for a rough guess
7900       cc+=gte_cycletab[source[i]&0x3f]/2;
7901     }
7902     else if(/*itype[i]==LOAD||itype[i]==STORE||*/itype[i]==C1LS) // load,store causes weird timing issues
7903     {
7904       cc+=2; // 2 cycle penalty (after CLOCK_DIVIDER)
7905     }
7906     else if(i>1&&itype[i]==STORE&&itype[i-1]==STORE&&itype[i-2]==STORE&&!bt[i])
7907     {
7908       cc+=4;
7909     }
7910     else if(itype[i]==C2LS)
7911     {
7912       cc+=4;
7913     }
7914 #endif
7915     else
7916     {
7917       cc++;
7918     }
7919
7920     if(!is_ds[i]) {
7921       regs[i].dirty=current.dirty;
7922       regs[i].isconst=current.isconst;
7923       memcpy(constmap[i],current_constmap,sizeof(current_constmap));
7924     }
7925     for(hr=0;hr<HOST_REGS;hr++) {
7926       if(hr!=EXCLUDE_REG&&regs[i].regmap[hr]>=0) {
7927         if(regmap_pre[i][hr]!=regs[i].regmap[hr]) {
7928           regs[i].wasconst&=~(1<<hr);
7929         }
7930       }
7931     }
7932     if(current.regmap[HOST_BTREG]==BTREG) current.regmap[HOST_BTREG]=-1;
7933     regs[i].waswritten=current.waswritten;
7934   }
7935
7936   /* Pass 4 - Cull unused host registers */
7937
7938   uint64_t nr=0;
7939
7940   for (i=slen-1;i>=0;i--)
7941   {
7942     int hr;
7943     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7944     {
7945       if(ba[i]<start || ba[i]>=(start+slen*4))
7946       {
7947         // Branch out of this block, don't need anything
7948         nr=0;
7949       }
7950       else
7951       {
7952         // Internal branch
7953         // Need whatever matches the target
7954         nr=0;
7955         int t=(ba[i]-start)>>2;
7956         for(hr=0;hr<HOST_REGS;hr++)
7957         {
7958           if(regs[i].regmap_entry[hr]>=0) {
7959             if(regs[i].regmap_entry[hr]==regs[t].regmap_entry[hr]) nr|=1<<hr;
7960           }
7961         }
7962       }
7963       // Conditional branch may need registers for following instructions
7964       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7965       {
7966         if(i<slen-2) {
7967           nr|=needed_reg[i+2];
7968           for(hr=0;hr<HOST_REGS;hr++)
7969           {
7970             if(regmap_pre[i+2][hr]>=0&&get_reg(regs[i+2].regmap_entry,regmap_pre[i+2][hr])<0) nr&=~(1<<hr);
7971             //if((regmap_entry[i+2][hr])>=0) if(!((nr>>hr)&1)) printf("%x-bogus(%d=%d)\n",start+i*4,hr,regmap_entry[i+2][hr]);
7972           }
7973         }
7974       }
7975       // Don't need stuff which is overwritten
7976       //if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
7977       //if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
7978       // Merge in delay slot
7979       for(hr=0;hr<HOST_REGS;hr++)
7980       {
7981         if(!likely[i]) {
7982           // These are overwritten unless the branch is "likely"
7983           // and the delay slot is nullified if not taken
7984           if(rt1[i+1]&&rt1[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7985           if(rt2[i+1]&&rt2[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7986         }
7987         if(rs1[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7988         if(rs2[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7989         if(rs1[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7990         if(rs2[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7991         if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) {
7992           if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
7993           if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
7994         }
7995       }
7996     }
7997     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
7998     {
7999       // SYSCALL instruction (software interrupt)
8000       nr=0;
8001     }
8002     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
8003     {
8004       // ERET instruction (return from interrupt)
8005       nr=0;
8006     }
8007     else // Non-branch
8008     {
8009       if(i<slen-1) {
8010         for(hr=0;hr<HOST_REGS;hr++) {
8011           if(regmap_pre[i+1][hr]>=0&&get_reg(regs[i+1].regmap_entry,regmap_pre[i+1][hr])<0) nr&=~(1<<hr);
8012           if(regs[i].regmap[hr]!=regmap_pre[i+1][hr]) nr&=~(1<<hr);
8013           if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
8014           if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
8015         }
8016       }
8017     }
8018     for(hr=0;hr<HOST_REGS;hr++)
8019     {
8020       // Overwritten registers are not needed
8021       if(rt1[i]&&rt1[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8022       if(rt2[i]&&rt2[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8023       if(FTEMP==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
8024       // Source registers are needed
8025       if(rs1[i]==regmap_pre[i][hr]) nr|=1<<hr;
8026       if(rs2[i]==regmap_pre[i][hr]) nr|=1<<hr;
8027       if(rs1[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8028       if(rs2[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
8029       if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) {
8030         if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
8031         if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
8032       }
8033       // Don't store a register immediately after writing it,
8034       // may prevent dual-issue.
8035       // But do so if this is a branch target, otherwise we
8036       // might have to load the register before the branch.
8037       if(i>0&&!bt[i]&&((regs[i].wasdirty>>hr)&1)) {
8038         if((regmap_pre[i][hr]>0&&!((unneeded_reg[i]>>regmap_pre[i][hr])&1))) {
8039           if(rt1[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
8040           if(rt2[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
8041         }
8042         if((regs[i].regmap_entry[hr]>0&&!((unneeded_reg[i]>>regs[i].regmap_entry[hr])&1))) {
8043           if(rt1[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
8044           if(rt2[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
8045         }
8046       }
8047     }
8048     // Cycle count is needed at branches.  Assume it is needed at the target too.
8049     if(i==0||bt[i]||itype[i]==CJUMP||itype[i]==SPAN) {
8050       if(regmap_pre[i][HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
8051       if(regs[i].regmap_entry[HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
8052     }
8053     // Save it
8054     needed_reg[i]=nr;
8055
8056     // Deallocate unneeded registers
8057     for(hr=0;hr<HOST_REGS;hr++)
8058     {
8059       if(!((nr>>hr)&1)) {
8060         if(regs[i].regmap_entry[hr]!=CCREG) regs[i].regmap_entry[hr]=-1;
8061         if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
8062            (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8063            (regs[i].regmap[hr]&63)!=PTEMP && (regs[i].regmap[hr]&63)!=CCREG)
8064         {
8065           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
8066           {
8067             if(likely[i]) {
8068               regs[i].regmap[hr]=-1;
8069               regs[i].isconst&=~(1<<hr);
8070               if(i<slen-2) {
8071                 regmap_pre[i+2][hr]=-1;
8072                 regs[i+2].wasconst&=~(1<<hr);
8073               }
8074             }
8075           }
8076         }
8077         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8078         {
8079           int map=0,temp=0;
8080           if(itype[i+1]==STORE || itype[i+1]==STORELR ||
8081              (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
8082             map=INVCP;
8083           }
8084           if(itype[i+1]==LOADLR || itype[i+1]==STORELR ||
8085              itype[i+1]==C1LS || itype[i+1]==C2LS)
8086             temp=FTEMP;
8087           if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
8088              (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8089              (regs[i].regmap[hr]&63)!=rt1[i+1] && (regs[i].regmap[hr]&63)!=rt2[i+1] &&
8090              regs[i].regmap[hr]!=rs1[i+1] && regs[i].regmap[hr]!=rs2[i+1] &&
8091              (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=PTEMP &&
8092              regs[i].regmap[hr]!=RHASH && regs[i].regmap[hr]!=RHTBL &&
8093              regs[i].regmap[hr]!=RTEMP && regs[i].regmap[hr]!=CCREG &&
8094              regs[i].regmap[hr]!=map )
8095           {
8096             regs[i].regmap[hr]=-1;
8097             regs[i].isconst&=~(1<<hr);
8098             if((branch_regs[i].regmap[hr]&63)!=rs1[i] && (branch_regs[i].regmap[hr]&63)!=rs2[i] &&
8099                (branch_regs[i].regmap[hr]&63)!=rt1[i] && (branch_regs[i].regmap[hr]&63)!=rt2[i] &&
8100                (branch_regs[i].regmap[hr]&63)!=rt1[i+1] && (branch_regs[i].regmap[hr]&63)!=rt2[i+1] &&
8101                branch_regs[i].regmap[hr]!=rs1[i+1] && branch_regs[i].regmap[hr]!=rs2[i+1] &&
8102                (branch_regs[i].regmap[hr]&63)!=temp && branch_regs[i].regmap[hr]!=PTEMP &&
8103                branch_regs[i].regmap[hr]!=RHASH && branch_regs[i].regmap[hr]!=RHTBL &&
8104                branch_regs[i].regmap[hr]!=RTEMP && branch_regs[i].regmap[hr]!=CCREG &&
8105                branch_regs[i].regmap[hr]!=map)
8106             {
8107               branch_regs[i].regmap[hr]=-1;
8108               branch_regs[i].regmap_entry[hr]=-1;
8109               if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
8110               {
8111                 if(!likely[i]&&i<slen-2) {
8112                   regmap_pre[i+2][hr]=-1;
8113                   regs[i+2].wasconst&=~(1<<hr);
8114                 }
8115               }
8116             }
8117           }
8118         }
8119         else
8120         {
8121           // Non-branch
8122           if(i>0)
8123           {
8124             int map=-1,temp=-1;
8125             if(itype[i]==STORE || itype[i]==STORELR ||
8126                       (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
8127               map=INVCP;
8128             }
8129             if(itype[i]==LOADLR || itype[i]==STORELR ||
8130                itype[i]==C1LS || itype[i]==C2LS)
8131               temp=FTEMP;
8132             if((regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
8133                regs[i].regmap[hr]!=rs1[i] && regs[i].regmap[hr]!=rs2[i] &&
8134                (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=map &&
8135                (itype[i]!=SPAN||regs[i].regmap[hr]!=CCREG))
8136             {
8137               if(i<slen-1&&!is_ds[i]) {
8138                 assert(regs[i].regmap[hr]<64);
8139                 if(regmap_pre[i+1][hr]!=-1 || regs[i].regmap[hr]>0)
8140                 if(regmap_pre[i+1][hr]!=regs[i].regmap[hr])
8141                 {
8142                   SysPrintf("fail: %x (%d %d!=%d)\n",start+i*4,hr,regmap_pre[i+1][hr],regs[i].regmap[hr]);
8143                   assert(regmap_pre[i+1][hr]==regs[i].regmap[hr]);
8144                 }
8145                 regmap_pre[i+1][hr]=-1;
8146                 if(regs[i+1].regmap_entry[hr]==CCREG) regs[i+1].regmap_entry[hr]=-1;
8147                 regs[i+1].wasconst&=~(1<<hr);
8148               }
8149               regs[i].regmap[hr]=-1;
8150               regs[i].isconst&=~(1<<hr);
8151             }
8152           }
8153         }
8154       } // if needed
8155     } // for hr
8156   }
8157
8158   /* Pass 5 - Pre-allocate registers */
8159
8160   // If a register is allocated during a loop, try to allocate it for the
8161   // entire loop, if possible.  This avoids loading/storing registers
8162   // inside of the loop.
8163
8164   signed char f_regmap[HOST_REGS];
8165   clear_all_regs(f_regmap);
8166   for(i=0;i<slen-1;i++)
8167   {
8168     if(itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8169     {
8170       if(ba[i]>=start && ba[i]<(start+i*4))
8171       if(itype[i+1]==NOP||itype[i+1]==MOV||itype[i+1]==ALU
8172       ||itype[i+1]==SHIFTIMM||itype[i+1]==IMM16||itype[i+1]==LOAD
8173       ||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS
8174       ||itype[i+1]==SHIFT||itype[i+1]==COP1
8175       ||itype[i+1]==COP2||itype[i+1]==C2LS||itype[i+1]==C2OP)
8176       {
8177         int t=(ba[i]-start)>>2;
8178         if(t>0&&(itype[t-1]!=UJUMP&&itype[t-1]!=RJUMP&&itype[t-1]!=CJUMP&&itype[t-1]!=SJUMP)) // loop_preload can't handle jumps into delay slots
8179         if(t<2||(itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||rt1[t-2]!=31) // call/ret assumes no registers allocated
8180         for(hr=0;hr<HOST_REGS;hr++)
8181         {
8182           if(regs[i].regmap[hr]>=0) {
8183             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8184               // dealloc old register
8185               int n;
8186               for(n=0;n<HOST_REGS;n++)
8187               {
8188                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8189               }
8190               // and alloc new one
8191               f_regmap[hr]=regs[i].regmap[hr];
8192             }
8193           }
8194           if(branch_regs[i].regmap[hr]>=0) {
8195             if(f_regmap[hr]!=branch_regs[i].regmap[hr]) {
8196               // dealloc old register
8197               int n;
8198               for(n=0;n<HOST_REGS;n++)
8199               {
8200                 if(f_regmap[n]==branch_regs[i].regmap[hr]) {f_regmap[n]=-1;}
8201               }
8202               // and alloc new one
8203               f_regmap[hr]=branch_regs[i].regmap[hr];
8204             }
8205           }
8206           if(ooo[i]) {
8207             if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1])
8208               f_regmap[hr]=branch_regs[i].regmap[hr];
8209           }else{
8210             if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1])
8211               f_regmap[hr]=branch_regs[i].regmap[hr];
8212           }
8213           // Avoid dirty->clean transition
8214           #ifdef DESTRUCTIVE_WRITEBACK
8215           if(t>0) if(get_reg(regmap_pre[t],f_regmap[hr])>=0) if((regs[t].wasdirty>>get_reg(regmap_pre[t],f_regmap[hr]))&1) f_regmap[hr]=-1;
8216           #endif
8217           // This check is only strictly required in the DESTRUCTIVE_WRITEBACK
8218           // case above, however it's always a good idea.  We can't hoist the
8219           // load if the register was already allocated, so there's no point
8220           // wasting time analyzing most of these cases.  It only "succeeds"
8221           // when the mapping was different and the load can be replaced with
8222           // a mov, which is of negligible benefit.  So such cases are
8223           // skipped below.
8224           if(f_regmap[hr]>0) {
8225             if(regs[t].regmap[hr]==f_regmap[hr]||(regs[t].regmap_entry[hr]<0&&get_reg(regmap_pre[t],f_regmap[hr])<0)) {
8226               int r=f_regmap[hr];
8227               for(j=t;j<=i;j++)
8228               {
8229                 //printf("Test %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8230                 if(r<34&&((unneeded_reg[j]>>r)&1)) break;
8231                 assert(r < 64);
8232                 if(regs[j].regmap[hr]==f_regmap[hr]&&(f_regmap[hr]&63)<TEMPREG) {
8233                   //printf("Hit %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8234                   int k;
8235                   if(regs[i].regmap[hr]==-1&&branch_regs[i].regmap[hr]==-1) {
8236                     if(get_reg(regs[i+2].regmap,f_regmap[hr])>=0) break;
8237                     if(r>63) {
8238                       if(get_reg(regs[i].regmap,r&63)<0) break;
8239                       if(get_reg(branch_regs[i].regmap,r&63)<0) break;
8240                     }
8241                     k=i;
8242                     while(k>1&&regs[k-1].regmap[hr]==-1) {
8243                       if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8244                         //printf("no free regs for store %x\n",start+(k-1)*4);
8245                         break;
8246                       }
8247                       if(get_reg(regs[k-1].regmap,f_regmap[hr])>=0) {
8248                         //printf("no-match due to different register\n");
8249                         break;
8250                       }
8251                       if(itype[k-2]==UJUMP||itype[k-2]==RJUMP||itype[k-2]==CJUMP||itype[k-2]==SJUMP) {
8252                         //printf("no-match due to branch\n");
8253                         break;
8254                       }
8255                       // call/ret fast path assumes no registers allocated
8256                       if(k>2&&(itype[k-3]==UJUMP||itype[k-3]==RJUMP)&&rt1[k-3]==31) {
8257                         break;
8258                       }
8259                       assert(r < 64);
8260                       k--;
8261                     }
8262                     if(regs[k-1].regmap[hr]==f_regmap[hr]&&regmap_pre[k][hr]==f_regmap[hr]) {
8263                       //printf("Extend r%d, %x ->\n",hr,start+k*4);
8264                       while(k<i) {
8265                         regs[k].regmap_entry[hr]=f_regmap[hr];
8266                         regs[k].regmap[hr]=f_regmap[hr];
8267                         regmap_pre[k+1][hr]=f_regmap[hr];
8268                         regs[k].wasdirty&=~(1<<hr);
8269                         regs[k].dirty&=~(1<<hr);
8270                         regs[k].wasdirty|=(1<<hr)&regs[k-1].dirty;
8271                         regs[k].dirty|=(1<<hr)&regs[k].wasdirty;
8272                         regs[k].wasconst&=~(1<<hr);
8273                         regs[k].isconst&=~(1<<hr);
8274                         k++;
8275                       }
8276                     }
8277                     else {
8278                       //printf("Fail Extend r%d, %x ->\n",hr,start+k*4);
8279                       break;
8280                     }
8281                     assert(regs[i-1].regmap[hr]==f_regmap[hr]);
8282                     if(regs[i-1].regmap[hr]==f_regmap[hr]&&regmap_pre[i][hr]==f_regmap[hr]) {
8283                       //printf("OK fill %x (r%d)\n",start+i*4,hr);
8284                       regs[i].regmap_entry[hr]=f_regmap[hr];
8285                       regs[i].regmap[hr]=f_regmap[hr];
8286                       regs[i].wasdirty&=~(1<<hr);
8287                       regs[i].dirty&=~(1<<hr);
8288                       regs[i].wasdirty|=(1<<hr)&regs[i-1].dirty;
8289                       regs[i].dirty|=(1<<hr)&regs[i-1].dirty;
8290                       regs[i].wasconst&=~(1<<hr);
8291                       regs[i].isconst&=~(1<<hr);
8292                       branch_regs[i].regmap_entry[hr]=f_regmap[hr];
8293                       branch_regs[i].wasdirty&=~(1<<hr);
8294                       branch_regs[i].wasdirty|=(1<<hr)&regs[i].dirty;
8295                       branch_regs[i].regmap[hr]=f_regmap[hr];
8296                       branch_regs[i].dirty&=~(1<<hr);
8297                       branch_regs[i].dirty|=(1<<hr)&regs[i].dirty;
8298                       branch_regs[i].wasconst&=~(1<<hr);
8299                       branch_regs[i].isconst&=~(1<<hr);
8300                       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
8301                         regmap_pre[i+2][hr]=f_regmap[hr];
8302                         regs[i+2].wasdirty&=~(1<<hr);
8303                         regs[i+2].wasdirty|=(1<<hr)&regs[i].dirty;
8304                       }
8305                     }
8306                   }
8307                   for(k=t;k<j;k++) {
8308                     // Alloc register clean at beginning of loop,
8309                     // but may dirty it in pass 6
8310                     regs[k].regmap_entry[hr]=f_regmap[hr];
8311                     regs[k].regmap[hr]=f_regmap[hr];
8312                     regs[k].dirty&=~(1<<hr);
8313                     regs[k].wasconst&=~(1<<hr);
8314                     regs[k].isconst&=~(1<<hr);
8315                     if(itype[k]==UJUMP||itype[k]==RJUMP||itype[k]==CJUMP||itype[k]==SJUMP) {
8316                       branch_regs[k].regmap_entry[hr]=f_regmap[hr];
8317                       branch_regs[k].regmap[hr]=f_regmap[hr];
8318                       branch_regs[k].dirty&=~(1<<hr);
8319                       branch_regs[k].wasconst&=~(1<<hr);
8320                       branch_regs[k].isconst&=~(1<<hr);
8321                       if(itype[k]!=RJUMP&&itype[k]!=UJUMP&&(source[k]>>16)!=0x1000) {
8322                         regmap_pre[k+2][hr]=f_regmap[hr];
8323                         regs[k+2].wasdirty&=~(1<<hr);
8324                       }
8325                     }
8326                     else
8327                     {
8328                       regmap_pre[k+1][hr]=f_regmap[hr];
8329                       regs[k+1].wasdirty&=~(1<<hr);
8330                     }
8331                   }
8332                   if(regs[j].regmap[hr]==f_regmap[hr])
8333                     regs[j].regmap_entry[hr]=f_regmap[hr];
8334                   break;
8335                 }
8336                 if(j==i) break;
8337                 if(regs[j].regmap[hr]>=0)
8338                   break;
8339                 if(get_reg(regs[j].regmap,f_regmap[hr])>=0) {
8340                   //printf("no-match due to different register\n");
8341                   break;
8342                 }
8343                 if(itype[j]==UJUMP||itype[j]==RJUMP||(source[j]>>16)==0x1000)
8344                 {
8345                   // Stop on unconditional branch
8346                   break;
8347                 }
8348                 if(itype[j]==CJUMP||itype[j]==SJUMP)
8349                 {
8350                   if(ooo[j]) {
8351                     if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1])
8352                       break;
8353                   }else{
8354                     if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1])
8355                       break;
8356                   }
8357                   if(get_reg(branch_regs[j].regmap,f_regmap[hr])>=0) {
8358                     //printf("no-match due to different register (branch)\n");
8359                     break;
8360                   }
8361                 }
8362                 if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8363                   //printf("No free regs for store %x\n",start+j*4);
8364                   break;
8365                 }
8366                 assert(f_regmap[hr]<64);
8367               }
8368             }
8369           }
8370         }
8371       }
8372     }else{
8373       // Non branch or undetermined branch target
8374       for(hr=0;hr<HOST_REGS;hr++)
8375       {
8376         if(hr!=EXCLUDE_REG) {
8377           if(regs[i].regmap[hr]>=0) {
8378             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8379               // dealloc old register
8380               int n;
8381               for(n=0;n<HOST_REGS;n++)
8382               {
8383                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8384               }
8385               // and alloc new one
8386               f_regmap[hr]=regs[i].regmap[hr];
8387             }
8388           }
8389         }
8390       }
8391       // Try to restore cycle count at branch targets
8392       if(bt[i]) {
8393         for(j=i;j<slen-1;j++) {
8394           if(regs[j].regmap[HOST_CCREG]!=-1) break;
8395           if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8396             //printf("no free regs for store %x\n",start+j*4);
8397             break;
8398           }
8399         }
8400         if(regs[j].regmap[HOST_CCREG]==CCREG) {
8401           int k=i;
8402           //printf("Extend CC, %x -> %x\n",start+k*4,start+j*4);
8403           while(k<j) {
8404             regs[k].regmap_entry[HOST_CCREG]=CCREG;
8405             regs[k].regmap[HOST_CCREG]=CCREG;
8406             regmap_pre[k+1][HOST_CCREG]=CCREG;
8407             regs[k+1].wasdirty|=1<<HOST_CCREG;
8408             regs[k].dirty|=1<<HOST_CCREG;
8409             regs[k].wasconst&=~(1<<HOST_CCREG);
8410             regs[k].isconst&=~(1<<HOST_CCREG);
8411             k++;
8412           }
8413           regs[j].regmap_entry[HOST_CCREG]=CCREG;
8414         }
8415         // Work backwards from the branch target
8416         if(j>i&&f_regmap[HOST_CCREG]==CCREG)
8417         {
8418           //printf("Extend backwards\n");
8419           int k;
8420           k=i;
8421           while(regs[k-1].regmap[HOST_CCREG]==-1) {
8422             if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8423               //printf("no free regs for store %x\n",start+(k-1)*4);
8424               break;
8425             }
8426             k--;
8427           }
8428           if(regs[k-1].regmap[HOST_CCREG]==CCREG) {
8429             //printf("Extend CC, %x ->\n",start+k*4);
8430             while(k<=i) {
8431               regs[k].regmap_entry[HOST_CCREG]=CCREG;
8432               regs[k].regmap[HOST_CCREG]=CCREG;
8433               regmap_pre[k+1][HOST_CCREG]=CCREG;
8434               regs[k+1].wasdirty|=1<<HOST_CCREG;
8435               regs[k].dirty|=1<<HOST_CCREG;
8436               regs[k].wasconst&=~(1<<HOST_CCREG);
8437               regs[k].isconst&=~(1<<HOST_CCREG);
8438               k++;
8439             }
8440           }
8441           else {
8442             //printf("Fail Extend CC, %x ->\n",start+k*4);
8443           }
8444         }
8445       }
8446       if(itype[i]!=STORE&&itype[i]!=STORELR&&itype[i]!=C1LS&&itype[i]!=SHIFT&&
8447          itype[i]!=NOP&&itype[i]!=MOV&&itype[i]!=ALU&&itype[i]!=SHIFTIMM&&
8448          itype[i]!=IMM16&&itype[i]!=LOAD&&itype[i]!=COP1)
8449       {
8450         memcpy(f_regmap,regs[i].regmap,sizeof(f_regmap));
8451       }
8452     }
8453   }
8454
8455   // This allocates registers (if possible) one instruction prior
8456   // to use, which can avoid a load-use penalty on certain CPUs.
8457   for(i=0;i<slen-1;i++)
8458   {
8459     if(!i||(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP))
8460     {
8461       if(!bt[i+1])
8462       {
8463         if(itype[i]==ALU||itype[i]==MOV||itype[i]==LOAD||itype[i]==SHIFTIMM||itype[i]==IMM16
8464            ||((itype[i]==COP1||itype[i]==COP2)&&opcode2[i]<3))
8465         {
8466           if(rs1[i+1]) {
8467             if((hr=get_reg(regs[i+1].regmap,rs1[i+1]))>=0)
8468             {
8469               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8470               {
8471                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8472                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8473                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8474                 regs[i].isconst&=~(1<<hr);
8475                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8476                 constmap[i][hr]=constmap[i+1][hr];
8477                 regs[i+1].wasdirty&=~(1<<hr);
8478                 regs[i].dirty&=~(1<<hr);
8479               }
8480             }
8481           }
8482           if(rs2[i+1]) {
8483             if((hr=get_reg(regs[i+1].regmap,rs2[i+1]))>=0)
8484             {
8485               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8486               {
8487                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8488                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8489                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8490                 regs[i].isconst&=~(1<<hr);
8491                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8492                 constmap[i][hr]=constmap[i+1][hr];
8493                 regs[i+1].wasdirty&=~(1<<hr);
8494                 regs[i].dirty&=~(1<<hr);
8495               }
8496             }
8497           }
8498           // Preload target address for load instruction (non-constant)
8499           if(itype[i+1]==LOAD&&rs1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8500             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8501             {
8502               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8503               {
8504                 regs[i].regmap[hr]=rs1[i+1];
8505                 regmap_pre[i+1][hr]=rs1[i+1];
8506                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8507                 regs[i].isconst&=~(1<<hr);
8508                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8509                 constmap[i][hr]=constmap[i+1][hr];
8510                 regs[i+1].wasdirty&=~(1<<hr);
8511                 regs[i].dirty&=~(1<<hr);
8512               }
8513             }
8514           }
8515           // Load source into target register
8516           if(lt1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8517             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8518             {
8519               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8520               {
8521                 regs[i].regmap[hr]=rs1[i+1];
8522                 regmap_pre[i+1][hr]=rs1[i+1];
8523                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8524                 regs[i].isconst&=~(1<<hr);
8525                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8526                 constmap[i][hr]=constmap[i+1][hr];
8527                 regs[i+1].wasdirty&=~(1<<hr);
8528                 regs[i].dirty&=~(1<<hr);
8529               }
8530             }
8531           }
8532           // Address for store instruction (non-constant)
8533           if(itype[i+1]==STORE||itype[i+1]==STORELR
8534              ||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SB/SH/SW/SD/SWC1/SDC1/SWC2/SDC2
8535             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8536               hr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1);
8537               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8538               else {regs[i+1].regmap[hr]=AGEN1+((i+1)&1);regs[i+1].isconst&=~(1<<hr);}
8539               assert(hr>=0);
8540               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8541               {
8542                 regs[i].regmap[hr]=rs1[i+1];
8543                 regmap_pre[i+1][hr]=rs1[i+1];
8544                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8545                 regs[i].isconst&=~(1<<hr);
8546                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8547                 constmap[i][hr]=constmap[i+1][hr];
8548                 regs[i+1].wasdirty&=~(1<<hr);
8549                 regs[i].dirty&=~(1<<hr);
8550               }
8551             }
8552           }
8553           if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) { // LWC1/LDC1, LWC2/LDC2
8554             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8555               int nr;
8556               hr=get_reg(regs[i+1].regmap,FTEMP);
8557               assert(hr>=0);
8558               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8559               {
8560                 regs[i].regmap[hr]=rs1[i+1];
8561                 regmap_pre[i+1][hr]=rs1[i+1];
8562                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8563                 regs[i].isconst&=~(1<<hr);
8564                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8565                 constmap[i][hr]=constmap[i+1][hr];
8566                 regs[i+1].wasdirty&=~(1<<hr);
8567                 regs[i].dirty&=~(1<<hr);
8568               }
8569               else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
8570               {
8571                 // move it to another register
8572                 regs[i+1].regmap[hr]=-1;
8573                 regmap_pre[i+2][hr]=-1;
8574                 regs[i+1].regmap[nr]=FTEMP;
8575                 regmap_pre[i+2][nr]=FTEMP;
8576                 regs[i].regmap[nr]=rs1[i+1];
8577                 regmap_pre[i+1][nr]=rs1[i+1];
8578                 regs[i+1].regmap_entry[nr]=rs1[i+1];
8579                 regs[i].isconst&=~(1<<nr);
8580                 regs[i+1].isconst&=~(1<<nr);
8581                 regs[i].dirty&=~(1<<nr);
8582                 regs[i+1].wasdirty&=~(1<<nr);
8583                 regs[i+1].dirty&=~(1<<nr);
8584                 regs[i+2].wasdirty&=~(1<<nr);
8585               }
8586             }
8587           }
8588           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR/*||itype[i+1]==C1LS||||itype[i+1]==C2LS*/) {
8589             if(itype[i+1]==LOAD)
8590               hr=get_reg(regs[i+1].regmap,rt1[i+1]);
8591             if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) // LWC1/LDC1, LWC2/LDC2
8592               hr=get_reg(regs[i+1].regmap,FTEMP);
8593             if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1/SWC2/SDC2
8594               hr=get_reg(regs[i+1].regmap,AGEN1+((i+1)&1));
8595               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8596             }
8597             if(hr>=0&&regs[i].regmap[hr]<0) {
8598               int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
8599               if(rs>=0&&((regs[i+1].wasconst>>rs)&1)) {
8600                 regs[i].regmap[hr]=AGEN1+((i+1)&1);
8601                 regmap_pre[i+1][hr]=AGEN1+((i+1)&1);
8602                 regs[i+1].regmap_entry[hr]=AGEN1+((i+1)&1);
8603                 regs[i].isconst&=~(1<<hr);
8604                 regs[i+1].wasdirty&=~(1<<hr);
8605                 regs[i].dirty&=~(1<<hr);
8606               }
8607             }
8608           }
8609         }
8610       }
8611     }
8612   }
8613
8614   /* Pass 6 - Optimize clean/dirty state */
8615   clean_registers(0,slen-1,1);
8616
8617   /* Pass 7 - Identify 32-bit registers */
8618   for (i=slen-1;i>=0;i--)
8619   {
8620     if(itype[i]==CJUMP||itype[i]==SJUMP)
8621     {
8622       // Conditional branch
8623       if((source[i]>>16)!=0x1000&&i<slen-2) {
8624         // Mark this address as a branch target since it may be called
8625         // upon return from interrupt
8626         bt[i+2]=1;
8627       }
8628     }
8629   }
8630
8631   if(itype[slen-1]==SPAN) {
8632     bt[slen-1]=1; // Mark as a branch target so instruction can restart after exception
8633   }
8634
8635 #ifdef DISASM
8636   /* Debug/disassembly */
8637   for(i=0;i<slen;i++)
8638   {
8639     printf("U:");
8640     int r;
8641     for(r=1;r<=CCREG;r++) {
8642       if((unneeded_reg[i]>>r)&1) {
8643         if(r==HIREG) printf(" HI");
8644         else if(r==LOREG) printf(" LO");
8645         else printf(" r%d",r);
8646       }
8647     }
8648     printf("\n");
8649     #if defined(__i386__) || defined(__x86_64__)
8650     printf("pre: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7]);
8651     #endif
8652     #ifdef __arm__
8653     printf("pre: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][4],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7],regmap_pre[i][8],regmap_pre[i][9],regmap_pre[i][10],regmap_pre[i][12]);
8654     #endif
8655     #if defined(__i386__) || defined(__x86_64__)
8656     printf("needs: ");
8657     if(needed_reg[i]&1) printf("eax ");
8658     if((needed_reg[i]>>1)&1) printf("ecx ");
8659     if((needed_reg[i]>>2)&1) printf("edx ");
8660     if((needed_reg[i]>>3)&1) printf("ebx ");
8661     if((needed_reg[i]>>5)&1) printf("ebp ");
8662     if((needed_reg[i]>>6)&1) printf("esi ");
8663     if((needed_reg[i]>>7)&1) printf("edi ");
8664     printf("\n");
8665     printf("entry: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7]);
8666     printf("dirty: ");
8667     if(regs[i].wasdirty&1) printf("eax ");
8668     if((regs[i].wasdirty>>1)&1) printf("ecx ");
8669     if((regs[i].wasdirty>>2)&1) printf("edx ");
8670     if((regs[i].wasdirty>>3)&1) printf("ebx ");
8671     if((regs[i].wasdirty>>5)&1) printf("ebp ");
8672     if((regs[i].wasdirty>>6)&1) printf("esi ");
8673     if((regs[i].wasdirty>>7)&1) printf("edi ");
8674     #endif
8675     #ifdef __arm__
8676     printf("entry: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[4],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7],regs[i].regmap_entry[8],regs[i].regmap_entry[9],regs[i].regmap_entry[10],regs[i].regmap_entry[12]);
8677     printf("dirty: ");
8678     if(regs[i].wasdirty&1) printf("r0 ");
8679     if((regs[i].wasdirty>>1)&1) printf("r1 ");
8680     if((regs[i].wasdirty>>2)&1) printf("r2 ");
8681     if((regs[i].wasdirty>>3)&1) printf("r3 ");
8682     if((regs[i].wasdirty>>4)&1) printf("r4 ");
8683     if((regs[i].wasdirty>>5)&1) printf("r5 ");
8684     if((regs[i].wasdirty>>6)&1) printf("r6 ");
8685     if((regs[i].wasdirty>>7)&1) printf("r7 ");
8686     if((regs[i].wasdirty>>8)&1) printf("r8 ");
8687     if((regs[i].wasdirty>>9)&1) printf("r9 ");
8688     if((regs[i].wasdirty>>10)&1) printf("r10 ");
8689     if((regs[i].wasdirty>>12)&1) printf("r12 ");
8690     #endif
8691     printf("\n");
8692     disassemble_inst(i);
8693     //printf ("ccadj[%d] = %d\n",i,ccadj[i]);
8694     #if defined(__i386__) || defined(__x86_64__)
8695     printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7]);
8696     if(regs[i].dirty&1) printf("eax ");
8697     if((regs[i].dirty>>1)&1) printf("ecx ");
8698     if((regs[i].dirty>>2)&1) printf("edx ");
8699     if((regs[i].dirty>>3)&1) printf("ebx ");
8700     if((regs[i].dirty>>5)&1) printf("ebp ");
8701     if((regs[i].dirty>>6)&1) printf("esi ");
8702     if((regs[i].dirty>>7)&1) printf("edi ");
8703     #endif
8704     #ifdef __arm__
8705     printf("r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[4],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7],regs[i].regmap[8],regs[i].regmap[9],regs[i].regmap[10],regs[i].regmap[12]);
8706     if(regs[i].dirty&1) printf("r0 ");
8707     if((regs[i].dirty>>1)&1) printf("r1 ");
8708     if((regs[i].dirty>>2)&1) printf("r2 ");
8709     if((regs[i].dirty>>3)&1) printf("r3 ");
8710     if((regs[i].dirty>>4)&1) printf("r4 ");
8711     if((regs[i].dirty>>5)&1) printf("r5 ");
8712     if((regs[i].dirty>>6)&1) printf("r6 ");
8713     if((regs[i].dirty>>7)&1) printf("r7 ");
8714     if((regs[i].dirty>>8)&1) printf("r8 ");
8715     if((regs[i].dirty>>9)&1) printf("r9 ");
8716     if((regs[i].dirty>>10)&1) printf("r10 ");
8717     if((regs[i].dirty>>12)&1) printf("r12 ");
8718     #endif
8719     printf("\n");
8720     if(regs[i].isconst) {
8721       printf("constants: ");
8722       #if defined(__i386__) || defined(__x86_64__)
8723       if(regs[i].isconst&1) printf("eax=%x ",(u_int)constmap[i][0]);
8724       if((regs[i].isconst>>1)&1) printf("ecx=%x ",(u_int)constmap[i][1]);
8725       if((regs[i].isconst>>2)&1) printf("edx=%x ",(u_int)constmap[i][2]);
8726       if((regs[i].isconst>>3)&1) printf("ebx=%x ",(u_int)constmap[i][3]);
8727       if((regs[i].isconst>>5)&1) printf("ebp=%x ",(u_int)constmap[i][5]);
8728       if((regs[i].isconst>>6)&1) printf("esi=%x ",(u_int)constmap[i][6]);
8729       if((regs[i].isconst>>7)&1) printf("edi=%x ",(u_int)constmap[i][7]);
8730       #endif
8731       #if defined(__arm__) || defined(__aarch64__)
8732       int r;
8733       for (r = 0; r < ARRAY_SIZE(constmap[i]); r++)
8734         if ((regs[i].isconst >> r) & 1)
8735           printf(" r%d=%x", r, (u_int)constmap[i][r]);
8736       #endif
8737       printf("\n");
8738     }
8739     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
8740       #if defined(__i386__) || defined(__x86_64__)
8741       printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
8742       if(branch_regs[i].dirty&1) printf("eax ");
8743       if((branch_regs[i].dirty>>1)&1) printf("ecx ");
8744       if((branch_regs[i].dirty>>2)&1) printf("edx ");
8745       if((branch_regs[i].dirty>>3)&1) printf("ebx ");
8746       if((branch_regs[i].dirty>>5)&1) printf("ebp ");
8747       if((branch_regs[i].dirty>>6)&1) printf("esi ");
8748       if((branch_regs[i].dirty>>7)&1) printf("edi ");
8749       #endif
8750       #ifdef __arm__
8751       printf("branch(%d): r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[4],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7],branch_regs[i].regmap[8],branch_regs[i].regmap[9],branch_regs[i].regmap[10],branch_regs[i].regmap[12]);
8752       if(branch_regs[i].dirty&1) printf("r0 ");
8753       if((branch_regs[i].dirty>>1)&1) printf("r1 ");
8754       if((branch_regs[i].dirty>>2)&1) printf("r2 ");
8755       if((branch_regs[i].dirty>>3)&1) printf("r3 ");
8756       if((branch_regs[i].dirty>>4)&1) printf("r4 ");
8757       if((branch_regs[i].dirty>>5)&1) printf("r5 ");
8758       if((branch_regs[i].dirty>>6)&1) printf("r6 ");
8759       if((branch_regs[i].dirty>>7)&1) printf("r7 ");
8760       if((branch_regs[i].dirty>>8)&1) printf("r8 ");
8761       if((branch_regs[i].dirty>>9)&1) printf("r9 ");
8762       if((branch_regs[i].dirty>>10)&1) printf("r10 ");
8763       if((branch_regs[i].dirty>>12)&1) printf("r12 ");
8764       #endif
8765     }
8766   }
8767 #endif // DISASM
8768
8769   /* Pass 8 - Assembly */
8770   linkcount=0;stubcount=0;
8771   ds=0;is_delayslot=0;
8772   u_int dirty_pre=0;
8773   void *beginning=start_block();
8774   if((u_int)addr&1) {
8775     ds=1;
8776     pagespan_ds();
8777   }
8778   void *instr_addr0_override = NULL;
8779
8780   if (start == 0x80030000) {
8781     // nasty hack for the fastbios thing
8782     // override block entry to this code
8783     instr_addr0_override = out;
8784     emit_movimm(start,0);
8785     // abuse io address var as a flag that we
8786     // have already returned here once
8787     emit_readword(&address,1);
8788     emit_writeword(0,&pcaddr);
8789     emit_writeword(0,&address);
8790     emit_cmp(0,1);
8791     #ifdef __aarch64__
8792     emit_jeq(out + 4*2);
8793     emit_far_jump(new_dyna_leave);
8794     #else
8795     emit_jne(new_dyna_leave);
8796     #endif
8797   }
8798   for(i=0;i<slen;i++)
8799   {
8800     //if(ds) printf("ds: ");
8801     disassemble_inst(i);
8802     if(ds) {
8803       ds=0; // Skip delay slot
8804       if(bt[i]) assem_debug("OOPS - branch into delay slot\n");
8805       instr_addr[i] = NULL;
8806     } else {
8807       speculate_register_values(i);
8808       #ifndef DESTRUCTIVE_WRITEBACK
8809       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8810       {
8811         wb_valid(regmap_pre[i],regs[i].regmap_entry,dirty_pre,regs[i].wasdirty,unneeded_reg[i]);
8812       }
8813       if((itype[i]==CJUMP||itype[i]==SJUMP)&&!likely[i]) {
8814         dirty_pre=branch_regs[i].dirty;
8815       }else{
8816         dirty_pre=regs[i].dirty;
8817       }
8818       #endif
8819       // write back
8820       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8821       {
8822         wb_invalidate(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,unneeded_reg[i]);
8823         loop_preload(regmap_pre[i],regs[i].regmap_entry);
8824       }
8825       // branch target entry point
8826       instr_addr[i] = out;
8827       assem_debug("<->\n");
8828       drc_dbg_emit_do_cmp(i);
8829
8830       // load regs
8831       if(regs[i].regmap_entry[HOST_CCREG]==CCREG&&regs[i].regmap[HOST_CCREG]!=CCREG)
8832         wb_register(CCREG,regs[i].regmap_entry,regs[i].wasdirty);
8833       load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i],rs2[i]);
8834       address_generation(i,&regs[i],regs[i].regmap_entry);
8835       load_consts(regmap_pre[i],regs[i].regmap,i);
8836       if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8837       {
8838         // Load the delay slot registers if necessary
8839         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i]&&(rs1[i+1]!=rt1[i]||rt1[i]==0))
8840           load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8841         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i]&&(rs2[i+1]!=rt1[i]||rt1[i]==0))
8842           load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8843         if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a)
8844           load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8845       }
8846       else if(i+1<slen)
8847       {
8848         // Preload registers for following instruction
8849         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
8850           if(rs1[i+1]!=rt1[i]&&rs1[i+1]!=rt2[i])
8851             load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8852         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
8853           if(rs2[i+1]!=rt1[i]&&rs2[i+1]!=rt2[i])
8854             load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8855       }
8856       // TODO: if(is_ooo(i)) address_generation(i+1);
8857       if(itype[i]==CJUMP)
8858         load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
8859       if(itype[i]==STORE||itype[i]==STORELR||(opcode[i]&0x3b)==0x39||(opcode[i]&0x3b)==0x3a)
8860         load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8861       // assemble
8862       switch(itype[i]) {
8863         case ALU:
8864           alu_assemble(i,&regs[i]);break;
8865         case IMM16:
8866           imm16_assemble(i,&regs[i]);break;
8867         case SHIFT:
8868           shift_assemble(i,&regs[i]);break;
8869         case SHIFTIMM:
8870           shiftimm_assemble(i,&regs[i]);break;
8871         case LOAD:
8872           load_assemble(i,&regs[i]);break;
8873         case LOADLR:
8874           loadlr_assemble(i,&regs[i]);break;
8875         case STORE:
8876           store_assemble(i,&regs[i]);break;
8877         case STORELR:
8878           storelr_assemble(i,&regs[i]);break;
8879         case COP0:
8880           cop0_assemble(i,&regs[i]);break;
8881         case COP1:
8882           cop1_assemble(i,&regs[i]);break;
8883         case C1LS:
8884           c1ls_assemble(i,&regs[i]);break;
8885         case COP2:
8886           cop2_assemble(i,&regs[i]);break;
8887         case C2LS:
8888           c2ls_assemble(i,&regs[i]);break;
8889         case C2OP:
8890           c2op_assemble(i,&regs[i]);break;
8891         case MULTDIV:
8892           multdiv_assemble(i,&regs[i]);break;
8893         case MOV:
8894           mov_assemble(i,&regs[i]);break;
8895         case SYSCALL:
8896           syscall_assemble(i,&regs[i]);break;
8897         case HLECALL:
8898           hlecall_assemble(i,&regs[i]);break;
8899         case INTCALL:
8900           intcall_assemble(i,&regs[i]);break;
8901         case UJUMP:
8902           ujump_assemble(i,&regs[i]);ds=1;break;
8903         case RJUMP:
8904           rjump_assemble(i,&regs[i]);ds=1;break;
8905         case CJUMP:
8906           cjump_assemble(i,&regs[i]);ds=1;break;
8907         case SJUMP:
8908           sjump_assemble(i,&regs[i]);ds=1;break;
8909         case SPAN:
8910           pagespan_assemble(i,&regs[i]);break;
8911       }
8912       if(itype[i]==UJUMP||itype[i]==RJUMP||(source[i]>>16)==0x1000)
8913         literal_pool(1024);
8914       else
8915         literal_pool_jumpover(256);
8916     }
8917   }
8918   //assert(itype[i-2]==UJUMP||itype[i-2]==RJUMP||(source[i-2]>>16)==0x1000);
8919   // If the block did not end with an unconditional branch,
8920   // add a jump to the next instruction.
8921   if(i>1) {
8922     if(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000&&itype[i-1]!=SPAN) {
8923       assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8924       assert(i==slen);
8925       if(itype[i-2]!=CJUMP&&itype[i-2]!=SJUMP) {
8926         store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8927         if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8928           emit_loadreg(CCREG,HOST_CCREG);
8929         emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8930       }
8931       else if(!likely[i-2])
8932       {
8933         store_regs_bt(branch_regs[i-2].regmap,branch_regs[i-2].dirty,start+i*4);
8934         assert(branch_regs[i-2].regmap[HOST_CCREG]==CCREG);
8935       }
8936       else
8937       {
8938         store_regs_bt(regs[i-2].regmap,regs[i-2].dirty,start+i*4);
8939         assert(regs[i-2].regmap[HOST_CCREG]==CCREG);
8940       }
8941       add_to_linker(out,start+i*4,0);
8942       emit_jmp(0);
8943     }
8944   }
8945   else
8946   {
8947     assert(i>0);
8948     assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8949     store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8950     if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8951       emit_loadreg(CCREG,HOST_CCREG);
8952     emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8953     add_to_linker(out,start+i*4,0);
8954     emit_jmp(0);
8955   }
8956
8957   // TODO: delay slot stubs?
8958   // Stubs
8959   for(i=0;i<stubcount;i++)
8960   {
8961     switch(stubs[i].type)
8962     {
8963       case LOADB_STUB:
8964       case LOADH_STUB:
8965       case LOADW_STUB:
8966       case LOADD_STUB:
8967       case LOADBU_STUB:
8968       case LOADHU_STUB:
8969         do_readstub(i);break;
8970       case STOREB_STUB:
8971       case STOREH_STUB:
8972       case STOREW_STUB:
8973       case STORED_STUB:
8974         do_writestub(i);break;
8975       case CC_STUB:
8976         do_ccstub(i);break;
8977       case INVCODE_STUB:
8978         do_invstub(i);break;
8979       case FP_STUB:
8980         do_cop1stub(i);break;
8981       case STORELR_STUB:
8982         do_unalignedwritestub(i);break;
8983     }
8984   }
8985
8986   if (instr_addr0_override)
8987     instr_addr[0] = instr_addr0_override;
8988
8989   /* Pass 9 - Linker */
8990   for(i=0;i<linkcount;i++)
8991   {
8992     assem_debug("%p -> %8x\n",link_addr[i].addr,link_addr[i].target);
8993     literal_pool(64);
8994     if (!link_addr[i].ext)
8995     {
8996       void *stub = out;
8997       void *addr = check_addr(link_addr[i].target);
8998       emit_extjump(link_addr[i].addr, link_addr[i].target);
8999       if (addr) {
9000         set_jump_target(link_addr[i].addr, addr);
9001         add_link(link_addr[i].target,stub);
9002       }
9003       else
9004         set_jump_target(link_addr[i].addr, stub);
9005     }
9006     else
9007     {
9008       // Internal branch
9009       int target=(link_addr[i].target-start)>>2;
9010       assert(target>=0&&target<slen);
9011       assert(instr_addr[target]);
9012       //#ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
9013       //set_jump_target_fillslot(link_addr[i].addr,instr_addr[target],link_addr[i].ext>>1);
9014       //#else
9015       set_jump_target(link_addr[i].addr, instr_addr[target]);
9016       //#endif
9017     }
9018   }
9019   // External Branch Targets (jump_in)
9020   if(copy+slen*4>(void *)shadow+sizeof(shadow)) copy=shadow;
9021   for(i=0;i<slen;i++)
9022   {
9023     if(bt[i]||i==0)
9024     {
9025       if(instr_addr[i]) // TODO - delay slots (=null)
9026       {
9027         u_int vaddr=start+i*4;
9028         u_int page=get_page(vaddr);
9029         u_int vpage=get_vpage(vaddr);
9030         literal_pool(256);
9031         {
9032           assem_debug("%p (%d) <- %8x\n",instr_addr[i],i,start+i*4);
9033           assem_debug("jump_in: %x\n",start+i*4);
9034           ll_add(jump_dirty+vpage,vaddr,out);
9035           void *entry_point = do_dirty_stub(i);
9036           ll_add_flags(jump_in+page,vaddr,state_rflags,entry_point);
9037           // If there was an existing entry in the hash table,
9038           // replace it with the new address.
9039           // Don't add new entries.  We'll insert the
9040           // ones that actually get used in check_addr().
9041           struct ht_entry *ht_bin = hash_table_get(vaddr);
9042           if (ht_bin->vaddr[0] == vaddr)
9043             ht_bin->tcaddr[0] = entry_point;
9044           if (ht_bin->vaddr[1] == vaddr)
9045             ht_bin->tcaddr[1] = entry_point;
9046         }
9047       }
9048     }
9049   }
9050   // Write out the literal pool if necessary
9051   literal_pool(0);
9052   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
9053   // Align code
9054   if(((u_int)out)&7) emit_addnop(13);
9055   #endif
9056   assert(out - (u_char *)beginning < MAX_OUTPUT_BLOCK_SIZE);
9057   //printf("shadow buffer: %p-%p\n",copy,(u_char *)copy+slen*4);
9058   memcpy(copy,source,slen*4);
9059   copy+=slen*4;
9060
9061   end_block(beginning);
9062
9063   // If we're within 256K of the end of the buffer,
9064   // start over from the beginning. (Is 256K enough?)
9065   if (out > ndrc->translation_cache + sizeof(ndrc->translation_cache) - MAX_OUTPUT_BLOCK_SIZE)
9066     out = ndrc->translation_cache;
9067
9068   // Trap writes to any of the pages we compiled
9069   for(i=start>>12;i<=(start+slen*4)>>12;i++) {
9070     invalid_code[i]=0;
9071   }
9072   inv_code_start=inv_code_end=~0;
9073
9074   // for PCSX we need to mark all mirrors too
9075   if(get_page(start)<(RAM_SIZE>>12))
9076     for(i=start>>12;i<=(start+slen*4)>>12;i++)
9077       invalid_code[((u_int)0x00000000>>12)|(i&0x1ff)]=
9078       invalid_code[((u_int)0x80000000>>12)|(i&0x1ff)]=
9079       invalid_code[((u_int)0xa0000000>>12)|(i&0x1ff)]=0;
9080
9081   /* Pass 10 - Free memory by expiring oldest blocks */
9082
9083   int end=(((out-ndrc->translation_cache)>>(TARGET_SIZE_2-16))+16384)&65535;
9084   while(expirep!=end)
9085   {
9086     int shift=TARGET_SIZE_2-3; // Divide into 8 blocks
9087     uintptr_t base=(uintptr_t)ndrc->translation_cache+((expirep>>13)<<shift); // Base address of this block
9088     inv_debug("EXP: Phase %d\n",expirep);
9089     switch((expirep>>11)&3)
9090     {
9091       case 0:
9092         // Clear jump_in and jump_dirty
9093         ll_remove_matching_addrs(jump_in+(expirep&2047),base,shift);
9094         ll_remove_matching_addrs(jump_dirty+(expirep&2047),base,shift);
9095         ll_remove_matching_addrs(jump_in+2048+(expirep&2047),base,shift);
9096         ll_remove_matching_addrs(jump_dirty+2048+(expirep&2047),base,shift);
9097         break;
9098       case 1:
9099         // Clear pointers
9100         ll_kill_pointers(jump_out[expirep&2047],base,shift);
9101         ll_kill_pointers(jump_out[(expirep&2047)+2048],base,shift);
9102         break;
9103       case 2:
9104         // Clear hash table
9105         for(i=0;i<32;i++) {
9106           struct ht_entry *ht_bin = &hash_table[((expirep&2047)<<5)+i];
9107           if (((uintptr_t)ht_bin->tcaddr[1]>>shift) == (base>>shift) ||
9108              (((uintptr_t)ht_bin->tcaddr[1]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
9109             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[1],ht_bin->tcaddr[1]);
9110             ht_bin->vaddr[1] = -1;
9111             ht_bin->tcaddr[1] = NULL;
9112           }
9113           if (((uintptr_t)ht_bin->tcaddr[0]>>shift) == (base>>shift) ||
9114              (((uintptr_t)ht_bin->tcaddr[0]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
9115             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[0],ht_bin->tcaddr[0]);
9116             ht_bin->vaddr[0] = ht_bin->vaddr[1];
9117             ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
9118             ht_bin->vaddr[1] = -1;
9119             ht_bin->tcaddr[1] = NULL;
9120           }
9121         }
9122         break;
9123       case 3:
9124         // Clear jump_out
9125         if((expirep&2047)==0)
9126           do_clear_cache();
9127         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
9128         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
9129         break;
9130     }
9131     expirep=(expirep+1)&65535;
9132   }
9133   return 0;
9134 }
9135
9136 // vim:shiftwidth=2:expandtab