libpcsxcore/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2011 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdlib.h>
  22 #include <stdint.h> //include for uint64_t
  23 #include <assert.h>
  24 #include <errno.h>
  25 #include <sys/mman.h>
  26 #ifdef __MACH__
  27 #include <libkern/OSCacheControl.h>
  28 #endif
  29 #ifdef _3DS
  30 #include <3ds_utils.h>
  31 #endif
  32 #ifdef VITA
  33 #include <psp2/kernel/sysmem.h>
  34 static int sceBlock;
  35 #endif
  36
  37 #include "new_dynarec_config.h"
  38 #include "../psxhle.h" //emulator interface
  39 #include "emu_if.h" //emulator interface
  40
  41 #ifndef ARRAY_SIZE
  42 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
  43 #endif
  44
  45 //#define DISASM
  46 //#define assem_debug printf
  47 //#define inv_debug printf
  48 #define assem_debug(...)
  49 #define inv_debug(...)
  50
  51 #ifdef __i386__
  52 #include "assem_x86.h"
  53 #endif
  54 #ifdef __x86_64__
  55 #include "assem_x64.h"
  56 #endif
  57 #ifdef __arm__
  58 #include "assem_arm.h"
  59 #endif
  60 #ifdef __aarch64__
  61 #include "assem_arm64.h"
  62 #endif
  63
  64 #define MAXBLOCK 4096
  65 #define MAX_OUTPUT_BLOCK_SIZE 262144
  66
  67 // stubs
  68 enum stub_type {
  69   CC_STUB = 1,
  70   FP_STUB = 2,
  71   LOADB_STUB = 3,
  72   LOADH_STUB = 4,
  73   LOADW_STUB = 5,
  74   LOADD_STUB = 6,
  75   LOADBU_STUB = 7,
  76   LOADHU_STUB = 8,
  77   STOREB_STUB = 9,
  78   STOREH_STUB = 10,
  79   STOREW_STUB = 11,
  80   STORED_STUB = 12,
  81   STORELR_STUB = 13,
  82   INVCODE_STUB = 14,
  83 };
  84
  85 struct regstat
  86 {
  87   signed char regmap_entry[HOST_REGS];
  88   signed char regmap[HOST_REGS];
  89   uint64_t wasdirty;
  90   uint64_t dirty;
  91   uint64_t u;
  92   u_int wasconst;
  93   u_int isconst;
  94   u_int loadedconst;             // host regs that have constants loaded
  95   u_int waswritten;              // MIPS regs that were used as store base before
  96 };
  97
  98 // note: asm depends on this layout
  99 struct ll_entry
 100 {
 101   u_int vaddr;
 102   u_int reg_sv_flags;
 103   void *addr;
 104   struct ll_entry *next;
 105 };
 106
 107 struct ht_entry
 108 {
 109   u_int vaddr[2];
 110   void *tcaddr[2];
 111 };
 112
 113 struct code_stub
 114 {
 115   enum stub_type type;
 116   void *addr;
 117   void *retaddr;
 118   u_int a;
 119   uintptr_t b;
 120   uintptr_t c;
 121   u_int d;
 122   u_int e;
 123 };
 124
 125 struct link_entry
 126 {
 127   void *addr;
 128   u_int target;
 129   u_int ext;
 130 };
 131
 132   // used by asm:
 133   u_char *out;
 134   struct ht_entry hash_table[65536]  __attribute__((aligned(16)));
 135   struct ll_entry *jump_in[4096] __attribute__((aligned(16)));
 136   struct ll_entry *jump_dirty[4096];
 137
 138   static struct ll_entry *jump_out[4096];
 139   static u_int start;
 140   static u_int *source;
 141   static char insn[MAXBLOCK][10];
 142   static u_char itype[MAXBLOCK];
 143   static u_char opcode[MAXBLOCK];
 144   static u_char opcode2[MAXBLOCK];
 145   static u_char bt[MAXBLOCK];
 146   static u_char rs1[MAXBLOCK];
 147   static u_char rs2[MAXBLOCK];
 148   static u_char rt1[MAXBLOCK];
 149   static u_char rt2[MAXBLOCK];
 150   static u_char us1[MAXBLOCK];
 151   static u_char us2[MAXBLOCK];
 152   static u_char dep1[MAXBLOCK];
 153   static u_char dep2[MAXBLOCK];
 154   static u_char lt1[MAXBLOCK];
 155   static uint64_t gte_rs[MAXBLOCK]; // gte: 32 data and 32 ctl regs
 156   static uint64_t gte_rt[MAXBLOCK];
 157   static uint64_t gte_unneeded[MAXBLOCK];
 158   static u_int smrv[32]; // speculated MIPS register values
 159   static u_int smrv_strong; // mask or regs that are likely to have correct values
 160   static u_int smrv_weak; // same, but somewhat less likely
 161   static u_int smrv_strong_next; // same, but after current insn executes
 162   static u_int smrv_weak_next;
 163   static int imm[MAXBLOCK];
 164   static u_int ba[MAXBLOCK];
 165   static char likely[MAXBLOCK];
 166   static char is_ds[MAXBLOCK];
 167   static char ooo[MAXBLOCK];
 168   static uint64_t unneeded_reg[MAXBLOCK];
 169   static uint64_t branch_unneeded_reg[MAXBLOCK];
 170   static signed char regmap_pre[MAXBLOCK][HOST_REGS];
 171   static uint64_t current_constmap[HOST_REGS];
 172   static uint64_t constmap[MAXBLOCK][HOST_REGS];
 173   static struct regstat regs[MAXBLOCK];
 174   static struct regstat branch_regs[MAXBLOCK];
 175   static signed char minimum_free_regs[MAXBLOCK];
 176   static u_int needed_reg[MAXBLOCK];
 177   static u_int wont_dirty[MAXBLOCK];
 178   static u_int will_dirty[MAXBLOCK];
 179   static int ccadj[MAXBLOCK];
 180   static int slen;
 181   static void *instr_addr[MAXBLOCK];
 182   static struct link_entry link_addr[MAXBLOCK];
 183   static int linkcount;
 184   static struct code_stub stubs[MAXBLOCK*3];
 185   static int stubcount;
 186   static u_int literals[1024][2];
 187   static int literalcount;
 188   static int is_delayslot;
 189   static char shadow[1048576]  __attribute__((aligned(16)));
 190   static void *copy;
 191   static int expirep;
 192   static u_int stop_after_jal;
 193 #ifndef RAM_FIXED
 194   static uintptr_t ram_offset;
 195 #else
 196   static const uintptr_t ram_offset=0;
 197 #endif
 198
 199   int new_dynarec_hacks;
 200   int new_dynarec_did_compile;
 201   extern u_char restore_candidate[512];
 202   extern int cycle_count;
 203
 204   /* registers that may be allocated */
 205   /* 1-31 gpr */
 206 #define HIREG 32 // hi
 207 #define LOREG 33 // lo
 208 //#define FSREG 34 // FPU status (FCSR)
 209 #define CSREG 35 // Coprocessor status
 210 #define CCREG 36 // Cycle count
 211 #define INVCP 37 // Pointer to invalid_code
 212 //#define MMREG 38 // Pointer to memory_map
 213 //#define ROREG 39 // ram offset (if rdram!=0x80000000)
 214 #define TEMPREG 40
 215 #define FTEMP 40 // FPU temporary register
 216 #define PTEMP 41 // Prefetch temporary register
 217 //#define TLREG 42 // TLB mapping offset
 218 #define RHASH 43 // Return address hash
 219 #define RHTBL 44 // Return address hash table address
 220 #define RTEMP 45 // JR/JALR address register
 221 #define MAXREG 45
 222 #define AGEN1 46 // Address generation temporary register
 223 //#define AGEN2 47 // Address generation temporary register
 224 //#define MGEN1 48 // Maptable address generation temporary register
 225 //#define MGEN2 49 // Maptable address generation temporary register
 226 #define BTREG 50 // Branch target temporary register
 227
 228   /* instruction types */
 229 #define NOP 0     // No operation
 230 #define LOAD 1    // Load
 231 #define STORE 2   // Store
 232 #define LOADLR 3  // Unaligned load
 233 #define STORELR 4 // Unaligned store
 234 #define MOV 5     // Move
 235 #define ALU 6     // Arithmetic/logic
 236 #define MULTDIV 7 // Multiply/divide
 237 #define SHIFT 8   // Shift by register
 238 #define SHIFTIMM 9// Shift by immediate
 239 #define IMM16 10  // 16-bit immediate
 240 #define RJUMP 11  // Unconditional jump to register
 241 #define UJUMP 12  // Unconditional jump
 242 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 243 #define SJUMP 14  // Conditional branch (regimm format)
 244 #define COP0 15   // Coprocessor 0
 245 #define COP1 16   // Coprocessor 1
 246 #define C1LS 17   // Coprocessor 1 load/store
 247 //#define FJUMP 18  // Conditional branch (floating point)
 248 //#define FLOAT 19  // Floating point unit
 249 //#define FCONV 20  // Convert integer to float
 250 //#define FCOMP 21  // Floating point compare (sets FSREG)
 251 #define SYSCALL 22// SYSCALL
 252 #define OTHER 23  // Other
 253 #define SPAN 24   // Branch/delay slot spans 2 pages
 254 #define NI 25     // Not implemented
 255 #define HLECALL 26// PCSX fake opcodes for HLE
 256 #define COP2 27   // Coprocessor 2 move
 257 #define C2LS 28   // Coprocessor 2 load/store
 258 #define C2OP 29   // Coprocessor 2 operation
 259 #define INTCALL 30// Call interpreter to handle rare corner cases
 260
 261   /* branch codes */
 262 #define TAKEN 1
 263 #define NOTTAKEN 2
 264 #define NULLDS 3
 265
 266 // asm linkage
 267 int new_recompile_block(int addr);
 268 void *get_addr_ht(u_int vaddr);
 269 void invalidate_block(u_int block);
 270 void invalidate_addr(u_int addr);
 271 void remove_hash(int vaddr);
 272 void dyna_linker();
 273 void dyna_linker_ds();
 274 void verify_code();
 275 void verify_code_vm();
 276 void verify_code_ds();
 277 void cc_interrupt();
 278 void fp_exception();
 279 void fp_exception_ds();
 280 void jump_syscall_hle();
 281 void jump_hlecall();
 282 void jump_intcall();
 283 void new_dyna_leave();
 284
 285 // Needed by assembler
 286 static void wb_register(signed char r,signed char regmap[],uint64_t dirty);
 287 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty);
 288 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr);
 289 static void load_all_regs(signed char i_regmap[]);
 290 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 291 static void load_regs_entry(int t);
 292 static void load_all_consts(signed char regmap[],u_int dirty,int i);
 293
 294 static int verify_dirty(u_int *ptr);
 295 static int get_final_value(int hr, int i, int *value);
 296 static void add_stub(enum stub_type type, void *addr, void *retaddr,
 297   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e);
 298 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
 299   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist);
 300 static void add_to_linker(void *addr, u_int target, int ext);
 301 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override);
 302
 303 static void mprotect_w_x(void *start, void *end, int is_x)
 304 {
 305 #ifdef NO_WRITE_EXEC
 306   #if defined(VITA)
 307   // *Open* enables write on all memory that was
 308   // allocated by sceKernelAllocMemBlockForVM()?
 309   if (is_x)
 310     sceKernelCloseVMDomain();
 311   else
 312     sceKernelOpenVMDomain();
 313   #else
 314   u_long mstart = (u_long)start & ~4095ul;
 315   u_long mend = (u_long)end;
 316   if (mprotect((void *)mstart, mend - mstart,
 317                PROT_READ | (is_x ? PROT_EXEC : PROT_WRITE)) != 0)
 318     SysPrintf("mprotect(%c) failed: %s\n", is_x ? 'x' : 'w', strerror(errno));
 319   #endif
 320 #endif
 321 }
 322
 323 static void start_tcache_write(void *start, void *end)
 324 {
 325   mprotect_w_x(start, end, 0);
 326 }
 327
 328 static void end_tcache_write(void *start, void *end)
 329 {
 330 #ifdef __arm__
 331   size_t len = (char *)end - (char *)start;
 332   #if   defined(__BLACKBERRY_QNX__)
 333   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);
 334   #elif defined(__MACH__)
 335   sys_cache_control(kCacheFunctionPrepareForExecution, start, len);
 336   #elif defined(VITA)
 337   sceKernelSyncVMDomain(sceBlock, start, len);
 338   #elif defined(_3DS)
 339   ctr_flush_invalidate_cache();
 340   #else
 341   __clear_cache(start, end);
 342   #endif
 343   (void)len;
 344 #else
 345   __clear_cache(start, end);
 346 #endif
 347
 348   mprotect_w_x(start, end, 1);
 349 }
 350
 351 static void *start_block(void)
 352 {
 353   u_char *end = out + MAX_OUTPUT_BLOCK_SIZE;
 354   if (end > translation_cache + (1<<TARGET_SIZE_2))
 355     end = translation_cache + (1<<TARGET_SIZE_2);
 356   start_tcache_write(out, end);
 357   return out;
 358 }
 359
 360 static void end_block(void *start)
 361 {
 362   end_tcache_write(start, out);
 363 }
 364
 365 //#define DEBUG_CYCLE_COUNT 1
 366
 367 #define NO_CYCLE_PENALTY_THR 12
 368
 369 int cycle_multiplier; // 100 for 1.0
 370
 371 static int CLOCK_ADJUST(int x)
 372 {
 373   int s=(x>>31)|1;
 374   return (x * cycle_multiplier + s * 50) / 100;
 375 }
 376
 377 static u_int get_page(u_int vaddr)
 378 {
 379   u_int page=vaddr&~0xe0000000;
 380   if (page < 0x1000000)
 381     page &= ~0x0e00000; // RAM mirrors
 382   page>>=12;
 383   if(page>2048) page=2048+(page&2047);
 384   return page;
 385 }
 386
 387 // no virtual mem in PCSX
 388 static u_int get_vpage(u_int vaddr)
 389 {
 390   return get_page(vaddr);
 391 }
 392
 393 static struct ht_entry *hash_table_get(u_int vaddr)
 394 {
 395   return &hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 396 }
 397
 398 static void hash_table_add(struct ht_entry *ht_bin, u_int vaddr, void *tcaddr)
 399 {
 400   ht_bin->vaddr[1] = ht_bin->vaddr[0];
 401   ht_bin->tcaddr[1] = ht_bin->tcaddr[0];
 402   ht_bin->vaddr[0] = vaddr;
 403   ht_bin->tcaddr[0] = tcaddr;
 404 }
 405
 406 // some messy ari64's code, seems to rely on unsigned 32bit overflow
 407 static int doesnt_expire_soon(void *tcaddr)
 408 {
 409   u_int diff = (u_int)((u_char *)tcaddr - out) << (32-TARGET_SIZE_2);
 410   return diff > (u_int)(0x60000000 + (MAX_OUTPUT_BLOCK_SIZE << (32-TARGET_SIZE_2)));
 411 }
 412
 413 // Get address from virtual address
 414 // This is called from the recompiled JR/JALR instructions
 415 void *get_addr(u_int vaddr)
 416 {
 417   u_int page=get_page(vaddr);
 418   u_int vpage=get_vpage(vaddr);
 419   struct ll_entry *head;
 420   //printf("TRACE: count=%d next=%d (get_addr %x,page %d)\n",Count,next_interupt,vaddr,page);
 421   head=jump_in[page];
 422   while(head!=NULL) {
 423     if(head->vaddr==vaddr) {
 424   //printf("TRACE: count=%d next=%d (get_addr match %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 425       hash_table_add(hash_table_get(vaddr), vaddr, head->addr);
 426       return head->addr;
 427     }
 428     head=head->next;
 429   }
 430   head=jump_dirty[vpage];
 431   while(head!=NULL) {
 432     if(head->vaddr==vaddr) {
 433       //printf("TRACE: count=%d next=%d (get_addr match dirty %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 434       // Don't restore blocks which are about to expire from the cache
 435       if (doesnt_expire_soon(head->addr))
 436       if (verify_dirty(head->addr)) {
 437         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 438         invalid_code[vaddr>>12]=0;
 439         inv_code_start=inv_code_end=~0;
 440         if(vpage<2048) {
 441           restore_candidate[vpage>>3]|=1<<(vpage&7);
 442         }
 443         else restore_candidate[page>>3]|=1<<(page&7);
 444         struct ht_entry *ht_bin = hash_table_get(vaddr);
 445         if (ht_bin->vaddr[0] == vaddr)
 446           ht_bin->tcaddr[0] = head->addr; // Replace existing entry
 447         else
 448           hash_table_add(ht_bin, vaddr, head->addr);
 449
 450         return head->addr;
 451       }
 452     }
 453     head=head->next;
 454   }
 455   //printf("TRACE: count=%d next=%d (get_addr no-match %x)\n",Count,next_interupt,vaddr);
 456   int r=new_recompile_block(vaddr);
 457   if(r==0) return get_addr(vaddr);
 458   // Execute in unmapped page, generate pagefault execption
 459   Status|=2;
 460   Cause=(vaddr<<31)|0x8;
 461   EPC=(vaddr&1)?vaddr-5:vaddr;
 462   BadVAddr=(vaddr&~1);
 463   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 464   EntryHi=BadVAddr&0xFFFFE000;
 465   return get_addr_ht(0x80000000);
 466 }
 467 // Look up address in hash table first
 468 void *get_addr_ht(u_int vaddr)
 469 {
 470   //printf("TRACE: count=%d next=%d (get_addr_ht %x)\n",Count,next_interupt,vaddr);
 471   const struct ht_entry *ht_bin = hash_table_get(vaddr);
 472   if (ht_bin->vaddr[0] == vaddr) return ht_bin->tcaddr[0];
 473   if (ht_bin->vaddr[1] == vaddr) return ht_bin->tcaddr[1];
 474   return get_addr(vaddr);
 475 }
 476
 477 void clear_all_regs(signed char regmap[])
 478 {
 479   int hr;
 480   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 481 }
 482
 483 signed char get_reg(signed char regmap[],int r)
 484 {
 485   int hr;
 486   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 487   return -1;
 488 }
 489
 490 // Find a register that is available for two consecutive cycles
 491 signed char get_reg2(signed char regmap1[],signed char regmap2[],int r)
 492 {
 493   int hr;
 494   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 495   return -1;
 496 }
 497
 498 int count_free_regs(signed char regmap[])
 499 {
 500   int count=0;
 501   int hr;
 502   for(hr=0;hr<HOST_REGS;hr++)
 503   {
 504     if(hr!=EXCLUDE_REG) {
 505       if(regmap[hr]<0) count++;
 506     }
 507   }
 508   return count;
 509 }
 510
 511 void dirty_reg(struct regstat *cur,signed char reg)
 512 {
 513   int hr;
 514   if(!reg) return;
 515   for (hr=0;hr<HOST_REGS;hr++) {
 516     if((cur->regmap[hr]&63)==reg) {
 517       cur->dirty|=1<<hr;
 518     }
 519   }
 520 }
 521
 522 void set_const(struct regstat *cur,signed char reg,uint64_t value)
 523 {
 524   int hr;
 525   if(!reg) return;
 526   for (hr=0;hr<HOST_REGS;hr++) {
 527     if(cur->regmap[hr]==reg) {
 528       cur->isconst|=1<<hr;
 529       current_constmap[hr]=value;
 530     }
 531     else if((cur->regmap[hr]^64)==reg) {
 532       cur->isconst|=1<<hr;
 533       current_constmap[hr]=value>>32;
 534     }
 535   }
 536 }
 537
 538 void clear_const(struct regstat *cur,signed char reg)
 539 {
 540   int hr;
 541   if(!reg) return;
 542   for (hr=0;hr<HOST_REGS;hr++) {
 543     if((cur->regmap[hr]&63)==reg) {
 544       cur->isconst&=~(1<<hr);
 545     }
 546   }
 547 }
 548
 549 int is_const(struct regstat *cur,signed char reg)
 550 {
 551   int hr;
 552   if(reg<0) return 0;
 553   if(!reg) return 1;
 554   for (hr=0;hr<HOST_REGS;hr++) {
 555     if((cur->regmap[hr]&63)==reg) {
 556       return (cur->isconst>>hr)&1;
 557     }
 558   }
 559   return 0;
 560 }
 561 uint64_t get_const(struct regstat *cur,signed char reg)
 562 {
 563   int hr;
 564   if(!reg) return 0;
 565   for (hr=0;hr<HOST_REGS;hr++) {
 566     if(cur->regmap[hr]==reg) {
 567       return current_constmap[hr];
 568     }
 569   }
 570   SysPrintf("Unknown constant in r%d\n",reg);
 571   exit(1);
 572 }
 573
 574 // Least soon needed registers
 575 // Look at the next ten instructions and see which registers
 576 // will be used.  Try not to reallocate these.
 577 void lsn(u_char hsn[], int i, int *preferred_reg)
 578 {
 579   int j;
 580   int b=-1;
 581   for(j=0;j<9;j++)
 582   {
 583     if(i+j>=slen) {
 584       j=slen-i-1;
 585       break;
 586     }
 587     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 588     {
 589       // Don't go past an unconditonal jump
 590       j++;
 591       break;
 592     }
 593   }
 594   for(;j>=0;j--)
 595   {
 596     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 597     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 598     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 599     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 600     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 601       // Stores can allocate zero
 602       hsn[rs1[i+j]]=j;
 603       hsn[rs2[i+j]]=j;
 604     }
 605     // On some architectures stores need invc_ptr
 606     #if defined(HOST_IMM8)
 607     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39 || (opcode[i+j]&0x3b)==0x3a) {
 608       hsn[INVCP]=j;
 609     }
 610     #endif
 611     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 612     {
 613       hsn[CCREG]=j;
 614       b=j;
 615     }
 616   }
 617   if(b>=0)
 618   {
 619     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 620     {
 621       // Follow first branch
 622       int t=(ba[i+b]-start)>>2;
 623       j=7-b;if(t+j>=slen) j=slen-t-1;
 624       for(;j>=0;j--)
 625       {
 626         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 627         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 628         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 629         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 630       }
 631     }
 632     // TODO: preferred register based on backward branch
 633   }
 634   // Delay slot should preferably not overwrite branch conditions or cycle count
 635   if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
 636     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 637     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 638     hsn[CCREG]=1;
 639     // ...or hash tables
 640     hsn[RHASH]=1;
 641     hsn[RHTBL]=1;
 642   }
 643   // Coprocessor load/store needs FTEMP, even if not declared
 644   if(itype[i]==C1LS||itype[i]==C2LS) {
 645     hsn[FTEMP]=0;
 646   }
 647   // Load L/R also uses FTEMP as a temporary register
 648   if(itype[i]==LOADLR) {
 649     hsn[FTEMP]=0;
 650   }
 651   // Also SWL/SWR/SDL/SDR
 652   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) {
 653     hsn[FTEMP]=0;
 654   }
 655   // Don't remove the miniht registers
 656   if(itype[i]==UJUMP||itype[i]==RJUMP)
 657   {
 658     hsn[RHASH]=0;
 659     hsn[RHTBL]=0;
 660   }
 661 }
 662
 663 // We only want to allocate registers if we're going to use them again soon
 664 int needed_again(int r, int i)
 665 {
 666   int j;
 667   int b=-1;
 668   int rn=10;
 669
 670   if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000))
 671   {
 672     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 673       return 0; // Don't need any registers if exiting the block
 674   }
 675   for(j=0;j<9;j++)
 676   {
 677     if(i+j>=slen) {
 678       j=slen-i-1;
 679       break;
 680     }
 681     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 682     {
 683       // Don't go past an unconditonal jump
 684       j++;
 685       break;
 686     }
 687     if(itype[i+j]==SYSCALL||itype[i+j]==HLECALL||itype[i+j]==INTCALL||((source[i+j]&0xfc00003f)==0x0d))
 688     {
 689       break;
 690     }
 691   }
 692   for(;j>=1;j--)
 693   {
 694     if(rs1[i+j]==r) rn=j;
 695     if(rs2[i+j]==r) rn=j;
 696     if((unneeded_reg[i+j]>>r)&1) rn=10;
 697     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 698     {
 699       b=j;
 700     }
 701   }
 702   /*
 703   if(b>=0)
 704   {
 705     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 706     {
 707       // Follow first branch
 708       int o=rn;
 709       int t=(ba[i+b]-start)>>2;
 710       j=7-b;if(t+j>=slen) j=slen-t-1;
 711       for(;j>=0;j--)
 712       {
 713         if(!((unneeded_reg[t+j]>>r)&1)) {
 714           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 715           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 716         }
 717         else rn=o;
 718       }
 719     }
 720   }*/
 721   if(rn<10) return 1;
 722   (void)b;
 723   return 0;
 724 }
 725
 726 // Try to match register allocations at the end of a loop with those
 727 // at the beginning
 728 int loop_reg(int i, int r, int hr)
 729 {
 730   int j,k;
 731   for(j=0;j<9;j++)
 732   {
 733     if(i+j>=slen) {
 734       j=slen-i-1;
 735       break;
 736     }
 737     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 738     {
 739       // Don't go past an unconditonal jump
 740       j++;
 741       break;
 742     }
 743   }
 744   k=0;
 745   if(i>0){
 746     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)
 747       k--;
 748   }
 749   for(;k<j;k++)
 750   {
 751     assert(r < 64);
 752     if((unneeded_reg[i+k]>>r)&1) return hr;
 753     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP))
 754     {
 755       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 756       {
 757         int t=(ba[i+k]-start)>>2;
 758         int reg=get_reg(regs[t].regmap_entry,r);
 759         if(reg>=0) return reg;
 760         //reg=get_reg(regs[t+1].regmap_entry,r);
 761         //if(reg>=0) return reg;
 762       }
 763     }
 764   }
 765   return hr;
 766 }
 767
 768
 769 // Allocate every register, preserving source/target regs
 770 void alloc_all(struct regstat *cur,int i)
 771 {
 772   int hr;
 773
 774   for(hr=0;hr<HOST_REGS;hr++) {
 775     if(hr!=EXCLUDE_REG) {
 776       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 777          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 778       {
 779         cur->regmap[hr]=-1;
 780         cur->dirty&=~(1<<hr);
 781       }
 782       // Don't need zeros
 783       if((cur->regmap[hr]&63)==0)
 784       {
 785         cur->regmap[hr]=-1;
 786         cur->dirty&=~(1<<hr);
 787       }
 788     }
 789   }
 790 }
 791
 792 #ifdef DRC_DBG
 793 extern void gen_interupt();
 794 extern void do_insn_cmp();
 795 #define FUNCNAME(f) { (intptr_t)f, " " #f }
 796 static const struct {
 797   intptr_t addr;
 798   const char *name;
 799 } function_names[] = {
 800   FUNCNAME(cc_interrupt),
 801   FUNCNAME(gen_interupt),
 802   FUNCNAME(get_addr_ht),
 803   FUNCNAME(get_addr),
 804   FUNCNAME(jump_handler_read8),
 805   FUNCNAME(jump_handler_read16),
 806   FUNCNAME(jump_handler_read32),
 807   FUNCNAME(jump_handler_write8),
 808   FUNCNAME(jump_handler_write16),
 809   FUNCNAME(jump_handler_write32),
 810   FUNCNAME(invalidate_addr),
 811   FUNCNAME(verify_code_vm),
 812   FUNCNAME(verify_code),
 813   FUNCNAME(jump_hlecall),
 814   FUNCNAME(jump_syscall_hle),
 815   FUNCNAME(new_dyna_leave),
 816   FUNCNAME(pcsx_mtc0),
 817   FUNCNAME(pcsx_mtc0_ds),
 818   FUNCNAME(do_insn_cmp),
 819 };
 820
 821 static const char *func_name(intptr_t a)
 822 {
 823   int i;
 824   for (i = 0; i < sizeof(function_names)/sizeof(function_names[0]); i++)
 825     if (function_names[i].addr == a)
 826       return function_names[i].name;
 827   return "";
 828 }
 829 #else
 830 #define func_name(x) ""
 831 #endif
 832
 833 #ifdef __i386__
 834 #include "assem_x86.c"
 835 #endif
 836 #ifdef __x86_64__
 837 #include "assem_x64.c"
 838 #endif
 839 #ifdef __arm__
 840 #include "assem_arm.c"
 841 #endif
 842 #ifdef __aarch64__
 843 #include "assem_arm64.c"
 844 #endif
 845
 846 // Add virtual address mapping to linked list
 847 void ll_add(struct ll_entry **head,int vaddr,void *addr)
 848 {
 849   struct ll_entry *new_entry;
 850   new_entry=malloc(sizeof(struct ll_entry));
 851   assert(new_entry!=NULL);
 852   new_entry->vaddr=vaddr;
 853   new_entry->reg_sv_flags=0;
 854   new_entry->addr=addr;
 855   new_entry->next=*head;
 856   *head=new_entry;
 857 }
 858
 859 void ll_add_flags(struct ll_entry **head,int vaddr,u_int reg_sv_flags,void *addr)
 860 {
 861   ll_add(head,vaddr,addr);
 862   (*head)->reg_sv_flags=reg_sv_flags;
 863 }
 864
 865 // Check if an address is already compiled
 866 // but don't return addresses which are about to expire from the cache
 867 void *check_addr(u_int vaddr)
 868 {
 869   struct ht_entry *ht_bin = hash_table_get(vaddr);
 870   size_t i;
 871   for (i = 0; i < ARRAY_SIZE(ht_bin->vaddr); i++) {
 872     if (ht_bin->vaddr[i] == vaddr)
 873       if (doesnt_expire_soon((u_char *)ht_bin->tcaddr[i] - MAX_OUTPUT_BLOCK_SIZE))
 874         if (isclean(ht_bin->tcaddr[i]))
 875           return ht_bin->tcaddr[i];
 876   }
 877   u_int page=get_page(vaddr);
 878   struct ll_entry *head;
 879   head=jump_in[page];
 880   while (head != NULL) {
 881     if (head->vaddr == vaddr) {
 882       if (doesnt_expire_soon(head->addr)) {
 883         // Update existing entry with current address
 884         if (ht_bin->vaddr[0] == vaddr) {
 885           ht_bin->tcaddr[0] = head->addr;
 886           return head->addr;
 887         }
 888         if (ht_bin->vaddr[1] == vaddr) {
 889           ht_bin->tcaddr[1] = head->addr;
 890           return head->addr;
 891         }
 892         // Insert into hash table with low priority.
 893         // Don't evict existing entries, as they are probably
 894         // addresses that are being accessed frequently.
 895         if (ht_bin->vaddr[0] == -1) {
 896           ht_bin->vaddr[0] = vaddr;
 897           ht_bin->tcaddr[0] = head->addr;
 898         }
 899         else if (ht_bin->vaddr[1] == -1) {
 900           ht_bin->vaddr[1] = vaddr;
 901           ht_bin->tcaddr[1] = head->addr;
 902         }
 903         return head->addr;
 904       }
 905     }
 906     head=head->next;
 907   }
 908   return 0;
 909 }
 910
 911 void remove_hash(int vaddr)
 912 {
 913   //printf("remove hash: %x\n",vaddr);
 914   struct ht_entry *ht_bin = hash_table_get(vaddr);
 915   if (ht_bin->vaddr[1] == vaddr) {
 916     ht_bin->vaddr[1] = -1;
 917     ht_bin->tcaddr[1] = NULL;
 918   }
 919   if (ht_bin->vaddr[0] == vaddr) {
 920     ht_bin->vaddr[0] = ht_bin->vaddr[1];
 921     ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
 922     ht_bin->vaddr[1] = -1;
 923     ht_bin->tcaddr[1] = NULL;
 924   }
 925 }
 926
 927 void ll_remove_matching_addrs(struct ll_entry **head,uintptr_t addr,int shift)
 928 {
 929   struct ll_entry *next;
 930   while(*head) {
 931     if(((uintptr_t)((*head)->addr)>>shift)==(addr>>shift) ||
 932        ((uintptr_t)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
 933     {
 934       inv_debug("EXP: Remove pointer to %p (%x)\n",(*head)->addr,(*head)->vaddr);
 935       remove_hash((*head)->vaddr);
 936       next=(*head)->next;
 937       free(*head);
 938       *head=next;
 939     }
 940     else
 941     {
 942       head=&((*head)->next);
 943     }
 944   }
 945 }
 946
 947 // Remove all entries from linked list
 948 void ll_clear(struct ll_entry **head)
 949 {
 950   struct ll_entry *cur;
 951   struct ll_entry *next;
 952   if((cur=*head)) {
 953     *head=0;
 954     while(cur) {
 955       next=cur->next;
 956       free(cur);
 957       cur=next;
 958     }
 959   }
 960 }
 961
 962 // Dereference the pointers and remove if it matches
 963 static void ll_kill_pointers(struct ll_entry *head,uintptr_t addr,int shift)
 964 {
 965   while(head) {
 966     uintptr_t ptr = (uintptr_t)get_pointer(head->addr);
 967     inv_debug("EXP: Lookup pointer to %lx at %p (%x)\n",(long)ptr,head->addr,head->vaddr);
 968     if(((ptr>>shift)==(addr>>shift)) ||
 969        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
 970     {
 971       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
 972       void *host_addr=find_extjump_insn(head->addr);
 973       #ifdef __arm__
 974         mark_clear_cache(host_addr);
 975       #endif
 976       set_jump_target(host_addr, head->addr);
 977     }
 978     head=head->next;
 979   }
 980 }
 981
 982 // This is called when we write to a compiled block (see do_invstub)
 983 void invalidate_page(u_int page)
 984 {
 985   struct ll_entry *head;
 986   struct ll_entry *next;
 987   head=jump_in[page];
 988   jump_in[page]=0;
 989   while(head!=NULL) {
 990     inv_debug("INVALIDATE: %x\n",head->vaddr);
 991     remove_hash(head->vaddr);
 992     next=head->next;
 993     free(head);
 994     head=next;
 995   }
 996   head=jump_out[page];
 997   jump_out[page]=0;
 998   while(head!=NULL) {
 999     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
1000     void *host_addr=find_extjump_insn(head->addr);
1001     #ifdef __arm__
1002       mark_clear_cache(host_addr);
1003     #endif
1004     set_jump_target(host_addr, head->addr);
1005     next=head->next;
1006     free(head);
1007     head=next;
1008   }
1009 }
1010
1011 static void invalidate_block_range(u_int block, u_int first, u_int last)
1012 {
1013   u_int page=get_page(block<<12);
1014   //printf("first=%d last=%d\n",first,last);
1015   invalidate_page(page);
1016   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1017   assert(last<page+5);
1018   // Invalidate the adjacent pages if a block crosses a 4K boundary
1019   while(first<page) {
1020     invalidate_page(first);
1021     first++;
1022   }
1023   for(first=page+1;first<last;first++) {
1024     invalidate_page(first);
1025   }
1026   #if defined(__arm__) || defined(__aarch64__)
1027     do_clear_cache();
1028   #endif
1029
1030   // Don't trap writes
1031   invalid_code[block]=1;
1032
1033   #ifdef USE_MINI_HT
1034   memset(mini_ht,-1,sizeof(mini_ht));
1035   #endif
1036 }
1037
1038 void invalidate_block(u_int block)
1039 {
1040   u_int page=get_page(block<<12);
1041   u_int vpage=get_vpage(block<<12);
1042   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1043   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1044   u_int first,last;
1045   first=last=page;
1046   struct ll_entry *head;
1047   head=jump_dirty[vpage];
1048   //printf("page=%d vpage=%d\n",page,vpage);
1049   while(head!=NULL) {
1050     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1051       u_char *start, *end;
1052       get_bounds(head->addr, &start, &end);
1053       //printf("start: %p end: %p\n", start, end);
1054       if (page < 2048 && start >= rdram && end < rdram+RAM_SIZE) {
1055         if (((start-rdram)>>12) <= page && ((end-1-rdram)>>12) >= page) {
1056           if ((((start-rdram)>>12)&2047) < first) first = ((start-rdram)>>12)&2047;
1057           if ((((end-1-rdram)>>12)&2047) > last)  last = ((end-1-rdram)>>12)&2047;
1058         }
1059       }
1060     }
1061     head=head->next;
1062   }
1063   invalidate_block_range(block,first,last);
1064 }
1065
1066 void invalidate_addr(u_int addr)
1067 {
1068   //static int rhits;
1069   // this check is done by the caller
1070   //if (inv_code_start<=addr&&addr<=inv_code_end) { rhits++; return; }
1071   u_int page=get_vpage(addr);
1072   if(page<2048) { // RAM
1073     struct ll_entry *head;
1074     u_int addr_min=~0, addr_max=0;
1075     u_int mask=RAM_SIZE-1;
1076     u_int addr_main=0x80000000|(addr&mask);
1077     int pg1;
1078     inv_code_start=addr_main&~0xfff;
1079     inv_code_end=addr_main|0xfff;
1080     pg1=page;
1081     if (pg1>0) {
1082       // must check previous page too because of spans..
1083       pg1--;
1084       inv_code_start-=0x1000;
1085     }
1086     for(;pg1<=page;pg1++) {
1087       for(head=jump_dirty[pg1];head!=NULL;head=head->next) {
1088         u_char *start_h, *end_h;
1089         u_int start, end;
1090         get_bounds(head->addr, &start_h, &end_h);
1091         start = (uintptr_t)start_h - ram_offset;
1092         end = (uintptr_t)end_h - ram_offset;
1093         if(start<=addr_main&&addr_main<end) {
1094           if(start<addr_min) addr_min=start;
1095           if(end>addr_max) addr_max=end;
1096         }
1097         else if(addr_main<start) {
1098           if(start<inv_code_end)
1099             inv_code_end=start-1;
1100         }
1101         else {
1102           if(end>inv_code_start)
1103             inv_code_start=end;
1104         }
1105       }
1106     }
1107     if (addr_min!=~0) {
1108       inv_debug("INV ADDR: %08x hit %08x-%08x\n", addr, addr_min, addr_max);
1109       inv_code_start=inv_code_end=~0;
1110       invalidate_block_range(addr>>12,(addr_min&mask)>>12,(addr_max&mask)>>12);
1111       return;
1112     }
1113     else {
1114       inv_code_start=(addr&~mask)|(inv_code_start&mask);
1115       inv_code_end=(addr&~mask)|(inv_code_end&mask);
1116       inv_debug("INV ADDR: %08x miss, inv %08x-%08x, sk %d\n", addr, inv_code_start, inv_code_end, 0);
1117       return;
1118     }
1119   }
1120   invalidate_block(addr>>12);
1121 }
1122
1123 // This is called when loading a save state.
1124 // Anything could have changed, so invalidate everything.
1125 void invalidate_all_pages()
1126 {
1127   u_int page;
1128   for(page=0;page<4096;page++)
1129     invalidate_page(page);
1130   for(page=0;page<1048576;page++)
1131     if(!invalid_code[page]) {
1132       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1133       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1134     }
1135   #ifdef USE_MINI_HT
1136   memset(mini_ht,-1,sizeof(mini_ht));
1137   #endif
1138 }
1139
1140 // Add an entry to jump_out after making a link
1141 void add_link(u_int vaddr,void *src)
1142 {
1143   u_int page=get_page(vaddr);
1144   inv_debug("add_link: %p -> %x (%d)\n",src,vaddr,page);
1145   int *ptr=(int *)(src+4);
1146   assert((*ptr&0x0fff0000)==0x059f0000);
1147   (void)ptr;
1148   ll_add(jump_out+page,vaddr,src);
1149   //void *ptr=get_pointer(src);
1150   //inv_debug("add_link: Pointer is to %p\n",ptr);
1151 }
1152
1153 // If a code block was found to be unmodified (bit was set in
1154 // restore_candidate) and it remains unmodified (bit is clear
1155 // in invalid_code) then move the entries for that 4K page from
1156 // the dirty list to the clean list.
1157 void clean_blocks(u_int page)
1158 {
1159   struct ll_entry *head;
1160   inv_debug("INV: clean_blocks page=%d\n",page);
1161   head=jump_dirty[page];
1162   while(head!=NULL) {
1163     if(!invalid_code[head->vaddr>>12]) {
1164       // Don't restore blocks which are about to expire from the cache
1165       if (doesnt_expire_soon(head->addr)) {
1166         if(verify_dirty(head->addr)) {
1167           u_char *start, *end;
1168           //printf("Possibly Restore %x (%p)\n",head->vaddr, head->addr);
1169           u_int i;
1170           u_int inv=0;
1171           get_bounds(head->addr, &start, &end);
1172           if (start - rdram < RAM_SIZE) {
1173             for (i = (start-rdram+0x80000000)>>12; i <= (end-1-rdram+0x80000000)>>12; i++) {
1174               inv|=invalid_code[i];
1175             }
1176           }
1177           else if((signed int)head->vaddr>=(signed int)0x80000000+RAM_SIZE) {
1178             inv=1;
1179           }
1180           if(!inv) {
1181             void *clean_addr = get_clean_addr(head->addr);
1182             if (doesnt_expire_soon(clean_addr)) {
1183               u_int ppage=page;
1184               inv_debug("INV: Restored %x (%p/%p)\n",head->vaddr, head->addr, clean_addr);
1185               //printf("page=%x, addr=%x\n",page,head->vaddr);
1186               //assert(head->vaddr>>12==(page|0x80000));
1187               ll_add_flags(jump_in+ppage,head->vaddr,head->reg_sv_flags,clean_addr);
1188               struct ht_entry *ht_bin = hash_table_get(head->vaddr);
1189               if (ht_bin->vaddr[0] == head->vaddr)
1190                 ht_bin->tcaddr[0] = clean_addr; // Replace existing entry
1191               if (ht_bin->vaddr[1] == head->vaddr)
1192                 ht_bin->tcaddr[1] = clean_addr; // Replace existing entry
1193             }
1194           }
1195         }
1196       }
1197     }
1198     head=head->next;
1199   }
1200 }
1201
1202 /* Register allocation */
1203
1204 // Note: registers are allocated clean (unmodified state)
1205 // if you intend to modify the register, you must call dirty_reg().
1206 static void alloc_reg(struct regstat *cur,int i,signed char reg)
1207 {
1208   int r,hr;
1209   int preferred_reg = (reg&7);
1210   if(reg==CCREG) preferred_reg=HOST_CCREG;
1211   if(reg==PTEMP||reg==FTEMP) preferred_reg=12;
1212
1213   // Don't allocate unused registers
1214   if((cur->u>>reg)&1) return;
1215
1216   // see if it's already allocated
1217   for(hr=0;hr<HOST_REGS;hr++)
1218   {
1219     if(cur->regmap[hr]==reg) return;
1220   }
1221
1222   // Keep the same mapping if the register was already allocated in a loop
1223   preferred_reg = loop_reg(i,reg,preferred_reg);
1224
1225   // Try to allocate the preferred register
1226   if(cur->regmap[preferred_reg]==-1) {
1227     cur->regmap[preferred_reg]=reg;
1228     cur->dirty&=~(1<<preferred_reg);
1229     cur->isconst&=~(1<<preferred_reg);
1230     return;
1231   }
1232   r=cur->regmap[preferred_reg];
1233   assert(r < 64);
1234   if((cur->u>>r)&1) {
1235     cur->regmap[preferred_reg]=reg;
1236     cur->dirty&=~(1<<preferred_reg);
1237     cur->isconst&=~(1<<preferred_reg);
1238     return;
1239   }
1240
1241   // Clear any unneeded registers
1242   // We try to keep the mapping consistent, if possible, because it
1243   // makes branches easier (especially loops).  So we try to allocate
1244   // first (see above) before removing old mappings.  If this is not
1245   // possible then go ahead and clear out the registers that are no
1246   // longer needed.
1247   for(hr=0;hr<HOST_REGS;hr++)
1248   {
1249     r=cur->regmap[hr];
1250     if(r>=0) {
1251       assert(r < 64);
1252       if((cur->u>>r)&1) {cur->regmap[hr]=-1;break;}
1253     }
1254   }
1255   // Try to allocate any available register, but prefer
1256   // registers that have not been used recently.
1257   if(i>0) {
1258     for(hr=0;hr<HOST_REGS;hr++) {
1259       if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1260         if(regs[i-1].regmap[hr]!=rs1[i-1]&&regs[i-1].regmap[hr]!=rs2[i-1]&&regs[i-1].regmap[hr]!=rt1[i-1]&&regs[i-1].regmap[hr]!=rt2[i-1]) {
1261           cur->regmap[hr]=reg;
1262           cur->dirty&=~(1<<hr);
1263           cur->isconst&=~(1<<hr);
1264           return;
1265         }
1266       }
1267     }
1268   }
1269   // Try to allocate any available register
1270   for(hr=0;hr<HOST_REGS;hr++) {
1271     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1272       cur->regmap[hr]=reg;
1273       cur->dirty&=~(1<<hr);
1274       cur->isconst&=~(1<<hr);
1275       return;
1276     }
1277   }
1278
1279   // Ok, now we have to evict someone
1280   // Pick a register we hopefully won't need soon
1281   u_char hsn[MAXREG+1];
1282   memset(hsn,10,sizeof(hsn));
1283   int j;
1284   lsn(hsn,i,&preferred_reg);
1285   //printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",cur->regmap[0],cur->regmap[1],cur->regmap[2],cur->regmap[3],cur->regmap[5],cur->regmap[6],cur->regmap[7]);
1286   //printf("hsn(%x): %d %d %d %d %d %d %d\n",start+i*4,hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1287   if(i>0) {
1288     // Don't evict the cycle count at entry points, otherwise the entry
1289     // stub will have to write it.
1290     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1291     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1292     for(j=10;j>=3;j--)
1293     {
1294       // Alloc preferred register if available
1295       if(hsn[r=cur->regmap[preferred_reg]&63]==j) {
1296         for(hr=0;hr<HOST_REGS;hr++) {
1297           // Evict both parts of a 64-bit register
1298           if((cur->regmap[hr]&63)==r) {
1299             cur->regmap[hr]=-1;
1300             cur->dirty&=~(1<<hr);
1301             cur->isconst&=~(1<<hr);
1302           }
1303         }
1304         cur->regmap[preferred_reg]=reg;
1305         return;
1306       }
1307       for(r=1;r<=MAXREG;r++)
1308       {
1309         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1310           for(hr=0;hr<HOST_REGS;hr++) {
1311             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1312               if(cur->regmap[hr]==r+64) {
1313                 cur->regmap[hr]=reg;
1314                 cur->dirty&=~(1<<hr);
1315                 cur->isconst&=~(1<<hr);
1316                 return;
1317               }
1318             }
1319           }
1320           for(hr=0;hr<HOST_REGS;hr++) {
1321             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1322               if(cur->regmap[hr]==r) {
1323                 cur->regmap[hr]=reg;
1324                 cur->dirty&=~(1<<hr);
1325                 cur->isconst&=~(1<<hr);
1326                 return;
1327               }
1328             }
1329           }
1330         }
1331       }
1332     }
1333   }
1334   for(j=10;j>=0;j--)
1335   {
1336     for(r=1;r<=MAXREG;r++)
1337     {
1338       if(hsn[r]==j) {
1339         for(hr=0;hr<HOST_REGS;hr++) {
1340           if(cur->regmap[hr]==r+64) {
1341             cur->regmap[hr]=reg;
1342             cur->dirty&=~(1<<hr);
1343             cur->isconst&=~(1<<hr);
1344             return;
1345           }
1346         }
1347         for(hr=0;hr<HOST_REGS;hr++) {
1348           if(cur->regmap[hr]==r) {
1349             cur->regmap[hr]=reg;
1350             cur->dirty&=~(1<<hr);
1351             cur->isconst&=~(1<<hr);
1352             return;
1353           }
1354         }
1355       }
1356     }
1357   }
1358   SysPrintf("This shouldn't happen (alloc_reg)");exit(1);
1359 }
1360
1361 // Allocate a temporary register.  This is done without regard to
1362 // dirty status or whether the register we request is on the unneeded list
1363 // Note: This will only allocate one register, even if called multiple times
1364 static void alloc_reg_temp(struct regstat *cur,int i,signed char reg)
1365 {
1366   int r,hr;
1367   int preferred_reg = -1;
1368
1369   // see if it's already allocated
1370   for(hr=0;hr<HOST_REGS;hr++)
1371   {
1372     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==reg) return;
1373   }
1374
1375   // Try to allocate any available register
1376   for(hr=HOST_REGS-1;hr>=0;hr--) {
1377     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1378       cur->regmap[hr]=reg;
1379       cur->dirty&=~(1<<hr);
1380       cur->isconst&=~(1<<hr);
1381       return;
1382     }
1383   }
1384
1385   // Find an unneeded register
1386   for(hr=HOST_REGS-1;hr>=0;hr--)
1387   {
1388     r=cur->regmap[hr];
1389     if(r>=0) {
1390       assert(r < 64);
1391       if((cur->u>>r)&1) {
1392         if(i==0||((unneeded_reg[i-1]>>r)&1)) {
1393           cur->regmap[hr]=reg;
1394           cur->dirty&=~(1<<hr);
1395           cur->isconst&=~(1<<hr);
1396           return;
1397         }
1398       }
1399     }
1400   }
1401
1402   // Ok, now we have to evict someone
1403   // Pick a register we hopefully won't need soon
1404   // TODO: we might want to follow unconditional jumps here
1405   // TODO: get rid of dupe code and make this into a function
1406   u_char hsn[MAXREG+1];
1407   memset(hsn,10,sizeof(hsn));
1408   int j;
1409   lsn(hsn,i,&preferred_reg);
1410   //printf("hsn: %d %d %d %d %d %d %d\n",hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1411   if(i>0) {
1412     // Don't evict the cycle count at entry points, otherwise the entry
1413     // stub will have to write it.
1414     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1415     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1416     for(j=10;j>=3;j--)
1417     {
1418       for(r=1;r<=MAXREG;r++)
1419       {
1420         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1421           for(hr=0;hr<HOST_REGS;hr++) {
1422             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1423               if(cur->regmap[hr]==r+64) {
1424                 cur->regmap[hr]=reg;
1425                 cur->dirty&=~(1<<hr);
1426                 cur->isconst&=~(1<<hr);
1427                 return;
1428               }
1429             }
1430           }
1431           for(hr=0;hr<HOST_REGS;hr++) {
1432             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1433               if(cur->regmap[hr]==r) {
1434                 cur->regmap[hr]=reg;
1435                 cur->dirty&=~(1<<hr);
1436                 cur->isconst&=~(1<<hr);
1437                 return;
1438               }
1439             }
1440           }
1441         }
1442       }
1443     }
1444   }
1445   for(j=10;j>=0;j--)
1446   {
1447     for(r=1;r<=MAXREG;r++)
1448     {
1449       if(hsn[r]==j) {
1450         for(hr=0;hr<HOST_REGS;hr++) {
1451           if(cur->regmap[hr]==r+64) {
1452             cur->regmap[hr]=reg;
1453             cur->dirty&=~(1<<hr);
1454             cur->isconst&=~(1<<hr);
1455             return;
1456           }
1457         }
1458         for(hr=0;hr<HOST_REGS;hr++) {
1459           if(cur->regmap[hr]==r) {
1460             cur->regmap[hr]=reg;
1461             cur->dirty&=~(1<<hr);
1462             cur->isconst&=~(1<<hr);
1463             return;
1464           }
1465         }
1466       }
1467     }
1468   }
1469   SysPrintf("This shouldn't happen");exit(1);
1470 }
1471
1472 static void mov_alloc(struct regstat *current,int i)
1473 {
1474   // Note: Don't need to actually alloc the source registers
1475   //alloc_reg(current,i,rs1[i]);
1476   alloc_reg(current,i,rt1[i]);
1477
1478   clear_const(current,rs1[i]);
1479   clear_const(current,rt1[i]);
1480   dirty_reg(current,rt1[i]);
1481 }
1482
1483 static void shiftimm_alloc(struct regstat *current,int i)
1484 {
1485   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1486   {
1487     if(rt1[i]) {
1488       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1489       else lt1[i]=rs1[i];
1490       alloc_reg(current,i,rt1[i]);
1491       dirty_reg(current,rt1[i]);
1492       if(is_const(current,rs1[i])) {
1493         int v=get_const(current,rs1[i]);
1494         if(opcode2[i]==0x00) set_const(current,rt1[i],v<<imm[i]);
1495         if(opcode2[i]==0x02) set_const(current,rt1[i],(u_int)v>>imm[i]);
1496         if(opcode2[i]==0x03) set_const(current,rt1[i],v>>imm[i]);
1497       }
1498       else clear_const(current,rt1[i]);
1499     }
1500   }
1501   else
1502   {
1503     clear_const(current,rs1[i]);
1504     clear_const(current,rt1[i]);
1505   }
1506
1507   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1508   {
1509     assert(0);
1510   }
1511   if(opcode2[i]==0x3c) // DSLL32
1512   {
1513     assert(0);
1514   }
1515   if(opcode2[i]==0x3e) // DSRL32
1516   {
1517     assert(0);
1518   }
1519   if(opcode2[i]==0x3f) // DSRA32
1520   {
1521     assert(0);
1522   }
1523 }
1524
1525 static void shift_alloc(struct regstat *current,int i)
1526 {
1527   if(rt1[i]) {
1528     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1529     {
1530       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1531       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1532       alloc_reg(current,i,rt1[i]);
1533       if(rt1[i]==rs2[i]) {
1534         alloc_reg_temp(current,i,-1);
1535         minimum_free_regs[i]=1;
1536       }
1537     } else { // DSLLV/DSRLV/DSRAV
1538       assert(0);
1539     }
1540     clear_const(current,rs1[i]);
1541     clear_const(current,rs2[i]);
1542     clear_const(current,rt1[i]);
1543     dirty_reg(current,rt1[i]);
1544   }
1545 }
1546
1547 static void alu_alloc(struct regstat *current,int i)
1548 {
1549   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1550     if(rt1[i]) {
1551       if(rs1[i]&&rs2[i]) {
1552         alloc_reg(current,i,rs1[i]);
1553         alloc_reg(current,i,rs2[i]);
1554       }
1555       else {
1556         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1557         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1558       }
1559       alloc_reg(current,i,rt1[i]);
1560     }
1561   }
1562   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1563     if(rt1[i]) {
1564       alloc_reg(current,i,rs1[i]);
1565       alloc_reg(current,i,rs2[i]);
1566       alloc_reg(current,i,rt1[i]);
1567     }
1568   }
1569   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1570     if(rt1[i]) {
1571       if(rs1[i]&&rs2[i]) {
1572         alloc_reg(current,i,rs1[i]);
1573         alloc_reg(current,i,rs2[i]);
1574       }
1575       else
1576       {
1577         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1578         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1579       }
1580       alloc_reg(current,i,rt1[i]);
1581     }
1582   }
1583   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1584     assert(0);
1585   }
1586   clear_const(current,rs1[i]);
1587   clear_const(current,rs2[i]);
1588   clear_const(current,rt1[i]);
1589   dirty_reg(current,rt1[i]);
1590 }
1591
1592 static void imm16_alloc(struct regstat *current,int i)
1593 {
1594   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1595   else lt1[i]=rs1[i];
1596   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1597   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1598     assert(0);
1599   }
1600   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1601     clear_const(current,rs1[i]);
1602     clear_const(current,rt1[i]);
1603   }
1604   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1605     if(is_const(current,rs1[i])) {
1606       int v=get_const(current,rs1[i]);
1607       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1608       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1609       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1610     }
1611     else clear_const(current,rt1[i]);
1612   }
1613   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1614     if(is_const(current,rs1[i])) {
1615       int v=get_const(current,rs1[i]);
1616       set_const(current,rt1[i],v+imm[i]);
1617     }
1618     else clear_const(current,rt1[i]);
1619   }
1620   else {
1621     set_const(current,rt1[i],((long long)((short)imm[i]))<<16); // LUI
1622   }
1623   dirty_reg(current,rt1[i]);
1624 }
1625
1626 static void load_alloc(struct regstat *current,int i)
1627 {
1628   clear_const(current,rt1[i]);
1629   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1630   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1631   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1632   if(rt1[i]&&!((current->u>>rt1[i])&1)) {
1633     alloc_reg(current,i,rt1[i]);
1634     assert(get_reg(current->regmap,rt1[i])>=0);
1635     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1636     {
1637       assert(0);
1638     }
1639     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1640     {
1641       assert(0);
1642     }
1643     dirty_reg(current,rt1[i]);
1644     // LWL/LWR need a temporary register for the old value
1645     if(opcode[i]==0x22||opcode[i]==0x26)
1646     {
1647       alloc_reg(current,i,FTEMP);
1648       alloc_reg_temp(current,i,-1);
1649       minimum_free_regs[i]=1;
1650     }
1651   }
1652   else
1653   {
1654     // Load to r0 or unneeded register (dummy load)
1655     // but we still need a register to calculate the address
1656     if(opcode[i]==0x22||opcode[i]==0x26)
1657     {
1658       alloc_reg(current,i,FTEMP); // LWL/LWR need another temporary
1659     }
1660     alloc_reg_temp(current,i,-1);
1661     minimum_free_regs[i]=1;
1662     if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1663     {
1664       assert(0);
1665     }
1666   }
1667 }
1668
1669 void store_alloc(struct regstat *current,int i)
1670 {
1671   clear_const(current,rs2[i]);
1672   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1673   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1674   alloc_reg(current,i,rs2[i]);
1675   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1676     assert(0);
1677   }
1678   #if defined(HOST_IMM8)
1679   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1680   else alloc_reg(current,i,INVCP);
1681   #endif
1682   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) { // SWL/SWL/SDL/SDR
1683     alloc_reg(current,i,FTEMP);
1684   }
1685   // We need a temporary register for address generation
1686   alloc_reg_temp(current,i,-1);
1687   minimum_free_regs[i]=1;
1688 }
1689
1690 void c1ls_alloc(struct regstat *current,int i)
1691 {
1692   //clear_const(current,rs1[i]); // FIXME
1693   clear_const(current,rt1[i]);
1694   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1695   alloc_reg(current,i,CSREG); // Status
1696   alloc_reg(current,i,FTEMP);
1697   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1698     assert(0);
1699   }
1700   #if defined(HOST_IMM8)
1701   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1702   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1703     alloc_reg(current,i,INVCP);
1704   #endif
1705   // We need a temporary register for address generation
1706   alloc_reg_temp(current,i,-1);
1707 }
1708
1709 void c2ls_alloc(struct regstat *current,int i)
1710 {
1711   clear_const(current,rt1[i]);
1712   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1713   alloc_reg(current,i,FTEMP);
1714   #if defined(HOST_IMM8)
1715   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1716   if((opcode[i]&0x3b)==0x3a) // SWC2/SDC2
1717     alloc_reg(current,i,INVCP);
1718   #endif
1719   // We need a temporary register for address generation
1720   alloc_reg_temp(current,i,-1);
1721   minimum_free_regs[i]=1;
1722 }
1723
1724 #ifndef multdiv_alloc
1725 void multdiv_alloc(struct regstat *current,int i)
1726 {
1727   //  case 0x18: MULT
1728   //  case 0x19: MULTU
1729   //  case 0x1A: DIV
1730   //  case 0x1B: DIVU
1731   //  case 0x1C: DMULT
1732   //  case 0x1D: DMULTU
1733   //  case 0x1E: DDIV
1734   //  case 0x1F: DDIVU
1735   clear_const(current,rs1[i]);
1736   clear_const(current,rs2[i]);
1737   if(rs1[i]&&rs2[i])
1738   {
1739     if((opcode2[i]&4)==0) // 32-bit
1740     {
1741       current->u&=~(1LL<<HIREG);
1742       current->u&=~(1LL<<LOREG);
1743       alloc_reg(current,i,HIREG);
1744       alloc_reg(current,i,LOREG);
1745       alloc_reg(current,i,rs1[i]);
1746       alloc_reg(current,i,rs2[i]);
1747       dirty_reg(current,HIREG);
1748       dirty_reg(current,LOREG);
1749     }
1750     else // 64-bit
1751     {
1752       assert(0);
1753     }
1754   }
1755   else
1756   {
1757     // Multiply by zero is zero.
1758     // MIPS does not have a divide by zero exception.
1759     // The result is undefined, we return zero.
1760     alloc_reg(current,i,HIREG);
1761     alloc_reg(current,i,LOREG);
1762     dirty_reg(current,HIREG);
1763     dirty_reg(current,LOREG);
1764   }
1765 }
1766 #endif
1767
1768 void cop0_alloc(struct regstat *current,int i)
1769 {
1770   if(opcode2[i]==0) // MFC0
1771   {
1772     if(rt1[i]) {
1773       clear_const(current,rt1[i]);
1774       alloc_all(current,i);
1775       alloc_reg(current,i,rt1[i]);
1776       dirty_reg(current,rt1[i]);
1777     }
1778   }
1779   else if(opcode2[i]==4) // MTC0
1780   {
1781     if(rs1[i]){
1782       clear_const(current,rs1[i]);
1783       alloc_reg(current,i,rs1[i]);
1784       alloc_all(current,i);
1785     }
1786     else {
1787       alloc_all(current,i); // FIXME: Keep r0
1788       current->u&=~1LL;
1789       alloc_reg(current,i,0);
1790     }
1791   }
1792   else
1793   {
1794     // TLBR/TLBWI/TLBWR/TLBP/ERET
1795     assert(opcode2[i]==0x10);
1796     alloc_all(current,i);
1797   }
1798   minimum_free_regs[i]=HOST_REGS;
1799 }
1800
1801 static void cop12_alloc(struct regstat *current,int i)
1802 {
1803   alloc_reg(current,i,CSREG); // Load status
1804   if(opcode2[i]<3) // MFC1/CFC1
1805   {
1806     if(rt1[i]){
1807       clear_const(current,rt1[i]);
1808       alloc_reg(current,i,rt1[i]);
1809       dirty_reg(current,rt1[i]);
1810     }
1811     alloc_reg_temp(current,i,-1);
1812   }
1813   else if(opcode2[i]>3) // MTC1/CTC1
1814   {
1815     if(rs1[i]){
1816       clear_const(current,rs1[i]);
1817       alloc_reg(current,i,rs1[i]);
1818     }
1819     else {
1820       current->u&=~1LL;
1821       alloc_reg(current,i,0);
1822     }
1823     alloc_reg_temp(current,i,-1);
1824   }
1825   minimum_free_regs[i]=1;
1826 }
1827
1828 void c2op_alloc(struct regstat *current,int i)
1829 {
1830   alloc_reg_temp(current,i,-1);
1831 }
1832
1833 void syscall_alloc(struct regstat *current,int i)
1834 {
1835   alloc_cc(current,i);
1836   dirty_reg(current,CCREG);
1837   alloc_all(current,i);
1838   minimum_free_regs[i]=HOST_REGS;
1839   current->isconst=0;
1840 }
1841
1842 void delayslot_alloc(struct regstat *current,int i)
1843 {
1844   switch(itype[i]) {
1845     case UJUMP:
1846     case CJUMP:
1847     case SJUMP:
1848     case RJUMP:
1849     case SYSCALL:
1850     case HLECALL:
1851     case SPAN:
1852       assem_debug("jump in the delay slot.  this shouldn't happen.\n");//exit(1);
1853       SysPrintf("Disabled speculative precompilation\n");
1854       stop_after_jal=1;
1855       break;
1856     case IMM16:
1857       imm16_alloc(current,i);
1858       break;
1859     case LOAD:
1860     case LOADLR:
1861       load_alloc(current,i);
1862       break;
1863     case STORE:
1864     case STORELR:
1865       store_alloc(current,i);
1866       break;
1867     case ALU:
1868       alu_alloc(current,i);
1869       break;
1870     case SHIFT:
1871       shift_alloc(current,i);
1872       break;
1873     case MULTDIV:
1874       multdiv_alloc(current,i);
1875       break;
1876     case SHIFTIMM:
1877       shiftimm_alloc(current,i);
1878       break;
1879     case MOV:
1880       mov_alloc(current,i);
1881       break;
1882     case COP0:
1883       cop0_alloc(current,i);
1884       break;
1885     case COP1:
1886     case COP2:
1887       cop12_alloc(current,i);
1888       break;
1889     case C1LS:
1890       c1ls_alloc(current,i);
1891       break;
1892     case C2LS:
1893       c2ls_alloc(current,i);
1894       break;
1895     case C2OP:
1896       c2op_alloc(current,i);
1897       break;
1898   }
1899 }
1900
1901 // Special case where a branch and delay slot span two pages in virtual memory
1902 static void pagespan_alloc(struct regstat *current,int i)
1903 {
1904   current->isconst=0;
1905   current->wasconst=0;
1906   regs[i].wasconst=0;
1907   minimum_free_regs[i]=HOST_REGS;
1908   alloc_all(current,i);
1909   alloc_cc(current,i);
1910   dirty_reg(current,CCREG);
1911   if(opcode[i]==3) // JAL
1912   {
1913     alloc_reg(current,i,31);
1914     dirty_reg(current,31);
1915   }
1916   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
1917   {
1918     alloc_reg(current,i,rs1[i]);
1919     if (rt1[i]!=0) {
1920       alloc_reg(current,i,rt1[i]);
1921       dirty_reg(current,rt1[i]);
1922     }
1923   }
1924   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
1925   {
1926     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1927     if(rs2[i]) alloc_reg(current,i,rs2[i]);
1928   }
1929   else
1930   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
1931   {
1932     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1933   }
1934   //else ...
1935 }
1936
1937 static void add_stub(enum stub_type type, void *addr, void *retaddr,
1938   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e)
1939 {
1940   assert(a < ARRAY_SIZE(stubs));
1941   stubs[stubcount].type = type;
1942   stubs[stubcount].addr = addr;
1943   stubs[stubcount].retaddr = retaddr;
1944   stubs[stubcount].a = a;
1945   stubs[stubcount].b = b;
1946   stubs[stubcount].c = c;
1947   stubs[stubcount].d = d;
1948   stubs[stubcount].e = e;
1949   stubcount++;
1950 }
1951
1952 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
1953   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist)
1954 {
1955   add_stub(type, addr, retaddr, i, addr_reg, (uintptr_t)i_regs, ccadj, reglist);
1956 }
1957
1958 // Write out a single register
1959 static void wb_register(signed char r,signed char regmap[],uint64_t dirty)
1960 {
1961   int hr;
1962   for(hr=0;hr<HOST_REGS;hr++) {
1963     if(hr!=EXCLUDE_REG) {
1964       if((regmap[hr]&63)==r) {
1965         if((dirty>>hr)&1) {
1966           assert(regmap[hr]<64);
1967           emit_storereg(r,hr);
1968         }
1969       }
1970     }
1971   }
1972 }
1973
1974 static void wb_valid(signed char pre[],signed char entry[],u_int dirty_pre,u_int dirty,uint64_t u)
1975 {
1976   //if(dirty_pre==dirty) return;
1977   int hr,reg;
1978   for(hr=0;hr<HOST_REGS;hr++) {
1979     if(hr!=EXCLUDE_REG) {
1980       reg=pre[hr];
1981       if(((~u)>>(reg&63))&1) {
1982         if(reg>0) {
1983           if(((dirty_pre&~dirty)>>hr)&1) {
1984             if(reg>0&&reg<34) {
1985               emit_storereg(reg,hr);
1986             }
1987             else if(reg>=64) {
1988               assert(0);
1989             }
1990           }
1991         }
1992       }
1993     }
1994   }
1995 }
1996
1997 void rlist()
1998 {
1999   int i;
2000   printf("TRACE: ");
2001   for(i=0;i<32;i++)
2002     printf("r%d:%8x%8x ",i,((int *)(reg+i))[1],((int *)(reg+i))[0]);
2003   printf("\n");
2004 }
2005
2006 void alu_assemble(int i,struct regstat *i_regs)
2007 {
2008   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2009     if(rt1[i]) {
2010       signed char s1,s2,t;
2011       t=get_reg(i_regs->regmap,rt1[i]);
2012       if(t>=0) {
2013         s1=get_reg(i_regs->regmap,rs1[i]);
2014         s2=get_reg(i_regs->regmap,rs2[i]);
2015         if(rs1[i]&&rs2[i]) {
2016           assert(s1>=0);
2017           assert(s2>=0);
2018           if(opcode2[i]&2) emit_sub(s1,s2,t);
2019           else emit_add(s1,s2,t);
2020         }
2021         else if(rs1[i]) {
2022           if(s1>=0) emit_mov(s1,t);
2023           else emit_loadreg(rs1[i],t);
2024         }
2025         else if(rs2[i]) {
2026           if(s2>=0) {
2027             if(opcode2[i]&2) emit_neg(s2,t);
2028             else emit_mov(s2,t);
2029           }
2030           else {
2031             emit_loadreg(rs2[i],t);
2032             if(opcode2[i]&2) emit_neg(t,t);
2033           }
2034         }
2035         else emit_zeroreg(t);
2036       }
2037     }
2038   }
2039   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2040     assert(0);
2041   }
2042   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2043     if(rt1[i]) {
2044       signed char s1l,s2l,t;
2045       {
2046         t=get_reg(i_regs->regmap,rt1[i]);
2047         //assert(t>=0);
2048         if(t>=0) {
2049           s1l=get_reg(i_regs->regmap,rs1[i]);
2050           s2l=get_reg(i_regs->regmap,rs2[i]);
2051           if(rs2[i]==0) // rx<r0
2052           {
2053             assert(s1l>=0);
2054             if(opcode2[i]==0x2a) // SLT
2055               emit_shrimm(s1l,31,t);
2056             else // SLTU (unsigned can not be less than zero)
2057               emit_zeroreg(t);
2058           }
2059           else if(rs1[i]==0) // r0<rx
2060           {
2061             assert(s2l>=0);
2062             if(opcode2[i]==0x2a) // SLT
2063               emit_set_gz32(s2l,t);
2064             else // SLTU (set if not zero)
2065               emit_set_nz32(s2l,t);
2066           }
2067           else{
2068             assert(s1l>=0);assert(s2l>=0);
2069             if(opcode2[i]==0x2a) // SLT
2070               emit_set_if_less32(s1l,s2l,t);
2071             else // SLTU
2072               emit_set_if_carry32(s1l,s2l,t);
2073           }
2074         }
2075       }
2076     }
2077   }
2078   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2079     if(rt1[i]) {
2080       signed char s1l,s2l,tl;
2081       tl=get_reg(i_regs->regmap,rt1[i]);
2082       {
2083         if(tl>=0) {
2084           s1l=get_reg(i_regs->regmap,rs1[i]);
2085           s2l=get_reg(i_regs->regmap,rs2[i]);
2086           if(rs1[i]&&rs2[i]) {
2087             assert(s1l>=0);
2088             assert(s2l>=0);
2089             if(opcode2[i]==0x24) { // AND
2090               emit_and(s1l,s2l,tl);
2091             } else
2092             if(opcode2[i]==0x25) { // OR
2093               emit_or(s1l,s2l,tl);
2094             } else
2095             if(opcode2[i]==0x26) { // XOR
2096               emit_xor(s1l,s2l,tl);
2097             } else
2098             if(opcode2[i]==0x27) { // NOR
2099               emit_or(s1l,s2l,tl);
2100               emit_not(tl,tl);
2101             }
2102           }
2103           else
2104           {
2105             if(opcode2[i]==0x24) { // AND
2106               emit_zeroreg(tl);
2107             } else
2108             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2109               if(rs1[i]){
2110                 if(s1l>=0) emit_mov(s1l,tl);
2111                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2112               }
2113               else
2114               if(rs2[i]){
2115                 if(s2l>=0) emit_mov(s2l,tl);
2116                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2117               }
2118               else emit_zeroreg(tl);
2119             } else
2120             if(opcode2[i]==0x27) { // NOR
2121               if(rs1[i]){
2122                 if(s1l>=0) emit_not(s1l,tl);
2123                 else {
2124                   emit_loadreg(rs1[i],tl);
2125                   emit_not(tl,tl);
2126                 }
2127               }
2128               else
2129               if(rs2[i]){
2130                 if(s2l>=0) emit_not(s2l,tl);
2131                 else {
2132                   emit_loadreg(rs2[i],tl);
2133                   emit_not(tl,tl);
2134                 }
2135               }
2136               else emit_movimm(-1,tl);
2137             }
2138           }
2139         }
2140       }
2141     }
2142   }
2143 }
2144
2145 void imm16_assemble(int i,struct regstat *i_regs)
2146 {
2147   if (opcode[i]==0x0f) { // LUI
2148     if(rt1[i]) {
2149       signed char t;
2150       t=get_reg(i_regs->regmap,rt1[i]);
2151       //assert(t>=0);
2152       if(t>=0) {
2153         if(!((i_regs->isconst>>t)&1))
2154           emit_movimm(imm[i]<<16,t);
2155       }
2156     }
2157   }
2158   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2159     if(rt1[i]) {
2160       signed char s,t;
2161       t=get_reg(i_regs->regmap,rt1[i]);
2162       s=get_reg(i_regs->regmap,rs1[i]);
2163       if(rs1[i]) {
2164         //assert(t>=0);
2165         //assert(s>=0);
2166         if(t>=0) {
2167           if(!((i_regs->isconst>>t)&1)) {
2168             if(s<0) {
2169               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2170               emit_addimm(t,imm[i],t);
2171             }else{
2172               if(!((i_regs->wasconst>>s)&1))
2173                 emit_addimm(s,imm[i],t);
2174               else
2175                 emit_movimm(constmap[i][s]+imm[i],t);
2176             }
2177           }
2178         }
2179       } else {
2180         if(t>=0) {
2181           if(!((i_regs->isconst>>t)&1))
2182             emit_movimm(imm[i],t);
2183         }
2184       }
2185     }
2186   }
2187   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2188     if(rt1[i]) {
2189       signed char sh,sl,th,tl;
2190       th=get_reg(i_regs->regmap,rt1[i]|64);
2191       tl=get_reg(i_regs->regmap,rt1[i]);
2192       sh=get_reg(i_regs->regmap,rs1[i]|64);
2193       sl=get_reg(i_regs->regmap,rs1[i]);
2194       if(tl>=0) {
2195         if(rs1[i]) {
2196           assert(sh>=0);
2197           assert(sl>=0);
2198           if(th>=0) {
2199             emit_addimm64_32(sh,sl,imm[i],th,tl);
2200           }
2201           else {
2202             emit_addimm(sl,imm[i],tl);
2203           }
2204         } else {
2205           emit_movimm(imm[i],tl);
2206           if(th>=0) emit_movimm(((signed int)imm[i])>>31,th);
2207         }
2208       }
2209     }
2210   }
2211   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2212     if(rt1[i]) {
2213       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2214       signed char sl,t;
2215       t=get_reg(i_regs->regmap,rt1[i]);
2216       sl=get_reg(i_regs->regmap,rs1[i]);
2217       //assert(t>=0);
2218       if(t>=0) {
2219         if(rs1[i]>0) {
2220             if(opcode[i]==0x0a) { // SLTI
2221               if(sl<0) {
2222                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2223                 emit_slti32(t,imm[i],t);
2224               }else{
2225                 emit_slti32(sl,imm[i],t);
2226               }
2227             }
2228             else { // SLTIU
2229               if(sl<0) {
2230                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2231                 emit_sltiu32(t,imm[i],t);
2232               }else{
2233                 emit_sltiu32(sl,imm[i],t);
2234               }
2235             }
2236         }else{
2237           // SLTI(U) with r0 is just stupid,
2238           // nonetheless examples can be found
2239           if(opcode[i]==0x0a) // SLTI
2240             if(0<imm[i]) emit_movimm(1,t);
2241             else emit_zeroreg(t);
2242           else // SLTIU
2243           {
2244             if(imm[i]) emit_movimm(1,t);
2245             else emit_zeroreg(t);
2246           }
2247         }
2248       }
2249     }
2250   }
2251   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2252     if(rt1[i]) {
2253       signed char sh,sl,th,tl;
2254       th=get_reg(i_regs->regmap,rt1[i]|64);
2255       tl=get_reg(i_regs->regmap,rt1[i]);
2256       sh=get_reg(i_regs->regmap,rs1[i]|64);
2257       sl=get_reg(i_regs->regmap,rs1[i]);
2258       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2259         if(opcode[i]==0x0c) //ANDI
2260         {
2261           if(rs1[i]) {
2262             if(sl<0) {
2263               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2264               emit_andimm(tl,imm[i],tl);
2265             }else{
2266               if(!((i_regs->wasconst>>sl)&1))
2267                 emit_andimm(sl,imm[i],tl);
2268               else
2269                 emit_movimm(constmap[i][sl]&imm[i],tl);
2270             }
2271           }
2272           else
2273             emit_zeroreg(tl);
2274           if(th>=0) emit_zeroreg(th);
2275         }
2276         else
2277         {
2278           if(rs1[i]) {
2279             if(sl<0) {
2280               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2281             }
2282             if(th>=0) {
2283               if(sh<0) {
2284                 emit_loadreg(rs1[i]|64,th);
2285               }else{
2286                 emit_mov(sh,th);
2287               }
2288             }
2289             if(opcode[i]==0x0d) { // ORI
2290               if(sl<0) {
2291                 emit_orimm(tl,imm[i],tl);
2292               }else{
2293                 if(!((i_regs->wasconst>>sl)&1))
2294                   emit_orimm(sl,imm[i],tl);
2295                 else
2296                   emit_movimm(constmap[i][sl]|imm[i],tl);
2297               }
2298             }
2299             if(opcode[i]==0x0e) { // XORI
2300               if(sl<0) {
2301                 emit_xorimm(tl,imm[i],tl);
2302               }else{
2303                 if(!((i_regs->wasconst>>sl)&1))
2304                   emit_xorimm(sl,imm[i],tl);
2305                 else
2306                   emit_movimm(constmap[i][sl]^imm[i],tl);
2307               }
2308             }
2309           }
2310           else {
2311             emit_movimm(imm[i],tl);
2312             if(th>=0) emit_zeroreg(th);
2313           }
2314         }
2315       }
2316     }
2317   }
2318 }
2319
2320 void shiftimm_assemble(int i,struct regstat *i_regs)
2321 {
2322   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2323   {
2324     if(rt1[i]) {
2325       signed char s,t;
2326       t=get_reg(i_regs->regmap,rt1[i]);
2327       s=get_reg(i_regs->regmap,rs1[i]);
2328       //assert(t>=0);
2329       if(t>=0&&!((i_regs->isconst>>t)&1)){
2330         if(rs1[i]==0)
2331         {
2332           emit_zeroreg(t);
2333         }
2334         else
2335         {
2336           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2337           if(imm[i]) {
2338             if(opcode2[i]==0) // SLL
2339             {
2340               emit_shlimm(s<0?t:s,imm[i],t);
2341             }
2342             if(opcode2[i]==2) // SRL
2343             {
2344               emit_shrimm(s<0?t:s,imm[i],t);
2345             }
2346             if(opcode2[i]==3) // SRA
2347             {
2348               emit_sarimm(s<0?t:s,imm[i],t);
2349             }
2350           }else{
2351             // Shift by zero
2352             if(s>=0 && s!=t) emit_mov(s,t);
2353           }
2354         }
2355       }
2356       //emit_storereg(rt1[i],t); //DEBUG
2357     }
2358   }
2359   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2360   {
2361     assert(0);
2362   }
2363   if(opcode2[i]==0x3c) // DSLL32
2364   {
2365     assert(0);
2366   }
2367   if(opcode2[i]==0x3e) // DSRL32
2368   {
2369     assert(0);
2370   }
2371   if(opcode2[i]==0x3f) // DSRA32
2372   {
2373     assert(0);
2374   }
2375 }
2376
2377 #ifndef shift_assemble
2378 void shift_assemble(int i,struct regstat *i_regs)
2379 {
2380   printf("Need shift_assemble for this architecture.\n");
2381   exit(1);
2382 }
2383 #endif
2384
2385 enum {
2386   MTYPE_8000 = 0,
2387   MTYPE_8020,
2388   MTYPE_0000,
2389   MTYPE_A000,
2390   MTYPE_1F80,
2391 };
2392
2393 static int get_ptr_mem_type(u_int a)
2394 {
2395   if(a < 0x00200000) {
2396     if(a<0x1000&&((start>>20)==0xbfc||(start>>24)==0xa0))
2397       // return wrong, must use memhandler for BIOS self-test to pass
2398       // 007 does similar stuff from a00 mirror, weird stuff
2399       return MTYPE_8000;
2400     return MTYPE_0000;
2401   }
2402   if(0x1f800000 <= a && a < 0x1f801000)
2403     return MTYPE_1F80;
2404   if(0x80200000 <= a && a < 0x80800000)
2405     return MTYPE_8020;
2406   if(0xa0000000 <= a && a < 0xa0200000)
2407     return MTYPE_A000;
2408   return MTYPE_8000;
2409 }
2410
2411 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override)
2412 {
2413   void *jaddr = NULL;
2414   int type=0;
2415   int mr=rs1[i];
2416   if(((smrv_strong|smrv_weak)>>mr)&1) {
2417     type=get_ptr_mem_type(smrv[mr]);
2418     //printf("set %08x @%08x r%d %d\n", smrv[mr], start+i*4, mr, type);
2419   }
2420   else {
2421     // use the mirror we are running on
2422     type=get_ptr_mem_type(start);
2423     //printf("set nospec   @%08x r%d %d\n", start+i*4, mr, type);
2424   }
2425
2426   if(type==MTYPE_8020) { // RAM 80200000+ mirror
2427     emit_andimm(addr,~0x00e00000,HOST_TEMPREG);
2428     addr=*addr_reg_override=HOST_TEMPREG;
2429     type=0;
2430   }
2431   else if(type==MTYPE_0000) { // RAM 0 mirror
2432     emit_orimm(addr,0x80000000,HOST_TEMPREG);
2433     addr=*addr_reg_override=HOST_TEMPREG;
2434     type=0;
2435   }
2436   else if(type==MTYPE_A000) { // RAM A mirror
2437     emit_andimm(addr,~0x20000000,HOST_TEMPREG);
2438     addr=*addr_reg_override=HOST_TEMPREG;
2439     type=0;
2440   }
2441   else if(type==MTYPE_1F80) { // scratchpad
2442     if (psxH == (void *)0x1f800000) {
2443       emit_addimm(addr,-0x1f800000,HOST_TEMPREG);
2444       emit_cmpimm(HOST_TEMPREG,0x1000);
2445       jaddr=out;
2446       emit_jc(0);
2447     }
2448     else {
2449       // do the usual RAM check, jump will go to the right handler
2450       type=0;
2451     }
2452   }
2453
2454   if(type==0)
2455   {
2456     emit_cmpimm(addr,RAM_SIZE);
2457     jaddr=out;
2458     #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2459     // Hint to branch predictor that the branch is unlikely to be taken
2460     if(rs1[i]>=28)
2461       emit_jno_unlikely(0);
2462     else
2463     #endif
2464       emit_jno(0);
2465     if(ram_offset!=0) {
2466       emit_addimm(addr,ram_offset,HOST_TEMPREG);
2467       addr=*addr_reg_override=HOST_TEMPREG;
2468     }
2469   }
2470
2471   return jaddr;
2472 }
2473
2474 static void load_assemble(int i,struct regstat *i_regs)
2475 {
2476   int s,th,tl,addr;
2477   int offset;
2478   void *jaddr=0;
2479   int memtarget=0,c=0;
2480   int fastload_reg_override=0;
2481   u_int hr,reglist=0;
2482   th=get_reg(i_regs->regmap,rt1[i]|64);
2483   tl=get_reg(i_regs->regmap,rt1[i]);
2484   s=get_reg(i_regs->regmap,rs1[i]);
2485   offset=imm[i];
2486   for(hr=0;hr<HOST_REGS;hr++) {
2487     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2488   }
2489   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2490   if(s>=0) {
2491     c=(i_regs->wasconst>>s)&1;
2492     if (c) {
2493       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2494     }
2495   }
2496   //printf("load_assemble: c=%d\n",c);
2497   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2498   // FIXME: Even if the load is a NOP, we should check for pagefaults...
2499   if((tl<0&&(!c||(((u_int)constmap[i][s]+offset)>>16)==0x1f80))
2500     ||rt1[i]==0) {
2501       // could be FIFO, must perform the read
2502       // ||dummy read
2503       assem_debug("(forced read)\n");
2504       tl=get_reg(i_regs->regmap,-1);
2505       assert(tl>=0);
2506   }
2507   if(offset||s<0||c) addr=tl;
2508   else addr=s;
2509   //if(tl<0) tl=get_reg(i_regs->regmap,-1);
2510  if(tl>=0) {
2511   //printf("load_assemble: c=%d\n",c);
2512   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2513   assert(tl>=0); // Even if the load is a NOP, we must check for pagefaults and I/O
2514   reglist&=~(1<<tl);
2515   if(th>=0) reglist&=~(1<<th);
2516   if(!c) {
2517     #ifdef R29_HACK
2518     // Strmnnrmn's speed hack
2519     if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
2520     #endif
2521     {
2522       jaddr=emit_fastpath_cmp_jump(i,addr,&fastload_reg_override);
2523     }
2524   }
2525   else if(ram_offset&&memtarget) {
2526     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2527     fastload_reg_override=HOST_TEMPREG;
2528   }
2529   int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg
2530   if (opcode[i]==0x20) { // LB
2531     if(!c||memtarget) {
2532       if(!dummy) {
2533         {
2534           int x=0,a=tl;
2535           if(!c) a=addr;
2536           if(fastload_reg_override) a=fastload_reg_override;
2537
2538           emit_movsbl_indexed(x,a,tl);
2539         }
2540       }
2541       if(jaddr)
2542         add_stub_r(LOADB_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2543     }
2544     else
2545       inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2546   }
2547   if (opcode[i]==0x21) { // LH
2548     if(!c||memtarget) {
2549       if(!dummy) {
2550         int x=0,a=tl;
2551         if(!c) a=addr;
2552         if(fastload_reg_override) a=fastload_reg_override;
2553         emit_movswl_indexed(x,a,tl);
2554       }
2555       if(jaddr)
2556         add_stub_r(LOADH_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2557     }
2558     else
2559       inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2560   }
2561   if (opcode[i]==0x23) { // LW
2562     if(!c||memtarget) {
2563       if(!dummy) {
2564         int a=addr;
2565         if(fastload_reg_override) a=fastload_reg_override;
2566         emit_readword_indexed(0,a,tl);
2567       }
2568       if(jaddr)
2569         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2570     }
2571     else
2572       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2573   }
2574   if (opcode[i]==0x24) { // LBU
2575     if(!c||memtarget) {
2576       if(!dummy) {
2577         int x=0,a=tl;
2578         if(!c) a=addr;
2579         if(fastload_reg_override) a=fastload_reg_override;
2580
2581         emit_movzbl_indexed(x,a,tl);
2582       }
2583       if(jaddr)
2584         add_stub_r(LOADBU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2585     }
2586     else
2587       inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2588   }
2589   if (opcode[i]==0x25) { // LHU
2590     if(!c||memtarget) {
2591       if(!dummy) {
2592         int x=0,a=tl;
2593         if(!c) a=addr;
2594         if(fastload_reg_override) a=fastload_reg_override;
2595         emit_movzwl_indexed(x,a,tl);
2596       }
2597       if(jaddr)
2598         add_stub_r(LOADHU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2599     }
2600     else
2601       inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2602   }
2603   if (opcode[i]==0x27) { // LWU
2604     assert(th>=0);
2605     if(!c||memtarget) {
2606       if(!dummy) {
2607         int a=addr;
2608         if(fastload_reg_override) a=fastload_reg_override;
2609         emit_readword_indexed(0,a,tl);
2610       }
2611       if(jaddr)
2612         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2613     }
2614     else {
2615       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2616     }
2617     emit_zeroreg(th);
2618   }
2619   if (opcode[i]==0x37) { // LD
2620     assert(0);
2621   }
2622  }
2623 }
2624
2625 #ifndef loadlr_assemble
2626 void loadlr_assemble(int i,struct regstat *i_regs)
2627 {
2628   printf("Need loadlr_assemble for this architecture.\n");
2629   exit(1);
2630 }
2631 #endif
2632
2633 void store_assemble(int i,struct regstat *i_regs)
2634 {
2635   int s,tl;
2636   int addr,temp;
2637   int offset;
2638   void *jaddr=0;
2639   enum stub_type type;
2640   int memtarget=0,c=0;
2641   int agr=AGEN1+(i&1);
2642   int faststore_reg_override=0;
2643   u_int hr,reglist=0;
2644   tl=get_reg(i_regs->regmap,rs2[i]);
2645   s=get_reg(i_regs->regmap,rs1[i]);
2646   temp=get_reg(i_regs->regmap,agr);
2647   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2648   offset=imm[i];
2649   if(s>=0) {
2650     c=(i_regs->wasconst>>s)&1;
2651     if(c) {
2652       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2653     }
2654   }
2655   assert(tl>=0);
2656   assert(temp>=0);
2657   for(hr=0;hr<HOST_REGS;hr++) {
2658     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2659   }
2660   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2661   if(offset||s<0||c) addr=temp;
2662   else addr=s;
2663   if(!c) {
2664     jaddr=emit_fastpath_cmp_jump(i,addr,&faststore_reg_override);
2665   }
2666   else if(ram_offset&&memtarget) {
2667     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2668     faststore_reg_override=HOST_TEMPREG;
2669   }
2670
2671   if (opcode[i]==0x28) { // SB
2672     if(!c||memtarget) {
2673       int x=0,a=temp;
2674       if(!c) a=addr;
2675       if(faststore_reg_override) a=faststore_reg_override;
2676       emit_writebyte_indexed(tl,x,a);
2677     }
2678     type=STOREB_STUB;
2679   }
2680   if (opcode[i]==0x29) { // SH
2681     if(!c||memtarget) {
2682       int x=0,a=temp;
2683       if(!c) a=addr;
2684       if(faststore_reg_override) a=faststore_reg_override;
2685       emit_writehword_indexed(tl,x,a);
2686     }
2687     type=STOREH_STUB;
2688   }
2689   if (opcode[i]==0x2B) { // SW
2690     if(!c||memtarget) {
2691       int a=addr;
2692       if(faststore_reg_override) a=faststore_reg_override;
2693       emit_writeword_indexed(tl,0,a);
2694     }
2695     type=STOREW_STUB;
2696   }
2697   if (opcode[i]==0x3F) { // SD
2698     assert(0);
2699     type=STORED_STUB;
2700   }
2701   if(jaddr) {
2702     // PCSX store handlers don't check invcode again
2703     reglist|=1<<addr;
2704     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2705     jaddr=0;
2706   }
2707   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
2708     if(!c||memtarget) {
2709       #ifdef DESTRUCTIVE_SHIFT
2710       // The x86 shift operation is 'destructive'; it overwrites the
2711       // source register, so we need to make a copy first and use that.
2712       addr=temp;
2713       #endif
2714       #if defined(HOST_IMM8)
2715       int ir=get_reg(i_regs->regmap,INVCP);
2716       assert(ir>=0);
2717       emit_cmpmem_indexedsr12_reg(ir,addr,1);
2718       #else
2719       emit_cmpmem_indexedsr12_imm(invalid_code,addr,1);
2720       #endif
2721       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2722       emit_callne(invalidate_addr_reg[addr]);
2723       #else
2724       void *jaddr2 = out;
2725       emit_jne(0);
2726       add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),addr,0,0,0);
2727       #endif
2728     }
2729   }
2730   u_int addr_val=constmap[i][s]+offset;
2731   if(jaddr) {
2732     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2733   } else if(c&&!memtarget) {
2734     inline_writestub(type,i,addr_val,i_regs->regmap,rs2[i],ccadj[i],reglist);
2735   }
2736   // basic current block modification detection..
2737   // not looking back as that should be in mips cache already
2738   if(c&&start+i*4<addr_val&&addr_val<start+slen*4) {
2739     SysPrintf("write to %08x hits block %08x, pc=%08x\n",addr_val,start,start+i*4);
2740     assert(i_regs->regmap==regs[i].regmap); // not delay slot
2741     if(i_regs->regmap==regs[i].regmap) {
2742       load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
2743       wb_dirtys(regs[i].regmap_entry,regs[i].wasdirty);
2744       emit_movimm(start+i*4+4,0);
2745       emit_writeword(0,&pcaddr);
2746       emit_jmp(do_interrupt);
2747     }
2748   }
2749 }
2750
2751 void storelr_assemble(int i,struct regstat *i_regs)
2752 {
2753   int s,tl;
2754   int temp;
2755   int offset;
2756   void *jaddr=0;
2757   void *case1, *case2, *case3;
2758   void *done0, *done1, *done2;
2759   int memtarget=0,c=0;
2760   int agr=AGEN1+(i&1);
2761   u_int hr,reglist=0;
2762   tl=get_reg(i_regs->regmap,rs2[i]);
2763   s=get_reg(i_regs->regmap,rs1[i]);
2764   temp=get_reg(i_regs->regmap,agr);
2765   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2766   offset=imm[i];
2767   if(s>=0) {
2768     c=(i_regs->isconst>>s)&1;
2769     if(c) {
2770       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2771     }
2772   }
2773   assert(tl>=0);
2774   for(hr=0;hr<HOST_REGS;hr++) {
2775     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2776   }
2777   assert(temp>=0);
2778   if(!c) {
2779     emit_cmpimm(s<0||offset?temp:s,RAM_SIZE);
2780     if(!offset&&s!=temp) emit_mov(s,temp);
2781     jaddr=out;
2782     emit_jno(0);
2783   }
2784   else
2785   {
2786     if(!memtarget||!rs1[i]) {
2787       jaddr=out;
2788       emit_jmp(0);
2789     }
2790   }
2791   emit_addimm_no_flags(ram_offset,temp);
2792
2793   if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
2794     assert(0);
2795   }
2796
2797   emit_xorimm(temp,3,temp);
2798   emit_testimm(temp,2);
2799   case2=out;
2800   emit_jne(0);
2801   emit_testimm(temp,1);
2802   case1=out;
2803   emit_jne(0);
2804   // 0
2805   if (opcode[i]==0x2A) { // SWL
2806     emit_writeword_indexed(tl,0,temp);
2807   }
2808   if (opcode[i]==0x2E) { // SWR
2809     emit_writebyte_indexed(tl,3,temp);
2810   }
2811   if (opcode[i]==0x2C) { // SDL
2812     assert(0);
2813   }
2814   if (opcode[i]==0x2D) { // SDR
2815     assert(0);
2816   }
2817   done0=out;
2818   emit_jmp(0);
2819   // 1
2820   set_jump_target(case1, out);
2821   if (opcode[i]==0x2A) { // SWL
2822     // Write 3 msb into three least significant bytes
2823     if(rs2[i]) emit_rorimm(tl,8,tl);
2824     emit_writehword_indexed(tl,-1,temp);
2825     if(rs2[i]) emit_rorimm(tl,16,tl);
2826     emit_writebyte_indexed(tl,1,temp);
2827     if(rs2[i]) emit_rorimm(tl,8,tl);
2828   }
2829   if (opcode[i]==0x2E) { // SWR
2830     // Write two lsb into two most significant bytes
2831     emit_writehword_indexed(tl,1,temp);
2832   }
2833   if (opcode[i]==0x2C) { // SDL
2834     assert(0);
2835   }
2836   if (opcode[i]==0x2D) { // SDR
2837     assert(0);
2838   }
2839   done1=out;
2840   emit_jmp(0);
2841   // 2
2842   set_jump_target(case2, out);
2843   emit_testimm(temp,1);
2844   case3=out;
2845   emit_jne(0);
2846   if (opcode[i]==0x2A) { // SWL
2847     // Write two msb into two least significant bytes
2848     if(rs2[i]) emit_rorimm(tl,16,tl);
2849     emit_writehword_indexed(tl,-2,temp);
2850     if(rs2[i]) emit_rorimm(tl,16,tl);
2851   }
2852   if (opcode[i]==0x2E) { // SWR
2853     // Write 3 lsb into three most significant bytes
2854     emit_writebyte_indexed(tl,-1,temp);
2855     if(rs2[i]) emit_rorimm(tl,8,tl);
2856     emit_writehword_indexed(tl,0,temp);
2857     if(rs2[i]) emit_rorimm(tl,24,tl);
2858   }
2859   if (opcode[i]==0x2C) { // SDL
2860     assert(0);
2861   }
2862   if (opcode[i]==0x2D) { // SDR
2863     assert(0);
2864   }
2865   done2=out;
2866   emit_jmp(0);
2867   // 3
2868   set_jump_target(case3, out);
2869   if (opcode[i]==0x2A) { // SWL
2870     // Write msb into least significant byte
2871     if(rs2[i]) emit_rorimm(tl,24,tl);
2872     emit_writebyte_indexed(tl,-3,temp);
2873     if(rs2[i]) emit_rorimm(tl,8,tl);
2874   }
2875   if (opcode[i]==0x2E) { // SWR
2876     // Write entire word
2877     emit_writeword_indexed(tl,-3,temp);
2878   }
2879   if (opcode[i]==0x2C) { // SDL
2880     assert(0);
2881   }
2882   if (opcode[i]==0x2D) { // SDR
2883     assert(0);
2884   }
2885   set_jump_target(done0, out);
2886   set_jump_target(done1, out);
2887   set_jump_target(done2, out);
2888   if (opcode[i]==0x2C) { // SDL
2889     assert(0);
2890   }
2891   if (opcode[i]==0x2D) { // SDR
2892     assert(0);
2893   }
2894   if(!c||!memtarget)
2895     add_stub_r(STORELR_STUB,jaddr,out,i,temp,i_regs,ccadj[i],reglist);
2896   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
2897     emit_addimm_no_flags(-ram_offset,temp);
2898     #if defined(HOST_IMM8)
2899     int ir=get_reg(i_regs->regmap,INVCP);
2900     assert(ir>=0);
2901     emit_cmpmem_indexedsr12_reg(ir,temp,1);
2902     #else
2903     emit_cmpmem_indexedsr12_imm(invalid_code,temp,1);
2904     #endif
2905     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2906     emit_callne(invalidate_addr_reg[temp]);
2907     #else
2908     void *jaddr2 = out;
2909     emit_jne(0);
2910     add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),temp,0,0,0);
2911     #endif
2912   }
2913 }
2914
2915 static void cop0_assemble(int i,struct regstat *i_regs)
2916 {
2917   if(opcode2[i]==0) // MFC0
2918   {
2919     signed char t=get_reg(i_regs->regmap,rt1[i]);
2920     u_int copr=(source[i]>>11)&0x1f;
2921     //assert(t>=0); // Why does this happen?  OOT is weird
2922     if(t>=0&&rt1[i]!=0) {
2923       emit_readword(&reg_cop0[copr],t);
2924     }
2925   }
2926   else if(opcode2[i]==4) // MTC0
2927   {
2928     signed char s=get_reg(i_regs->regmap,rs1[i]);
2929     char copr=(source[i]>>11)&0x1f;
2930     assert(s>=0);
2931     wb_register(rs1[i],i_regs->regmap,i_regs->dirty);
2932     if(copr==9||copr==11||copr==12||copr==13) {
2933       emit_readword(&last_count,HOST_TEMPREG);
2934       emit_loadreg(CCREG,HOST_CCREG); // TODO: do proper reg alloc
2935       emit_add(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
2936       emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
2937       emit_writeword(HOST_CCREG,&Count);
2938     }
2939     // What a mess.  The status register (12) can enable interrupts,
2940     // so needs a special case to handle a pending interrupt.
2941     // The interrupt must be taken immediately, because a subsequent
2942     // instruction might disable interrupts again.
2943     if(copr==12||copr==13) {
2944       if (is_delayslot) {
2945         // burn cycles to cause cc_interrupt, which will
2946         // reschedule next_interupt. Relies on CCREG from above.
2947         assem_debug("MTC0 DS %d\n", copr);
2948         emit_writeword(HOST_CCREG,&last_count);
2949         emit_movimm(0,HOST_CCREG);
2950         emit_storereg(CCREG,HOST_CCREG);
2951         emit_loadreg(rs1[i],1);
2952         emit_movimm(copr,0);
2953         emit_call(pcsx_mtc0_ds);
2954         emit_loadreg(rs1[i],s);
2955         return;
2956       }
2957       emit_movimm(start+i*4+4,HOST_TEMPREG);
2958       emit_writeword(HOST_TEMPREG,&pcaddr);
2959       emit_movimm(0,HOST_TEMPREG);
2960       emit_writeword(HOST_TEMPREG,&pending_exception);
2961     }
2962     //else if(copr==12&&is_delayslot) emit_call((int)MTC0_R12);
2963     //else
2964     if(s==HOST_CCREG)
2965       emit_loadreg(rs1[i],1);
2966     else if(s!=1)
2967       emit_mov(s,1);
2968     emit_movimm(copr,0);
2969     emit_call(pcsx_mtc0);
2970     if(copr==9||copr==11||copr==12||copr==13) {
2971       emit_readword(&Count,HOST_CCREG);
2972       emit_readword(&next_interupt,HOST_TEMPREG);
2973       emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
2974       emit_sub(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
2975       emit_writeword(HOST_TEMPREG,&last_count);
2976       emit_storereg(CCREG,HOST_CCREG);
2977     }
2978     if(copr==12||copr==13) {
2979       assert(!is_delayslot);
2980       emit_readword(&pending_exception,14);
2981       emit_test(14,14);
2982       emit_jne(&do_interrupt);
2983     }
2984     emit_loadreg(rs1[i],s);
2985     if(get_reg(i_regs->regmap,rs1[i]|64)>=0)
2986       emit_loadreg(rs1[i]|64,get_reg(i_regs->regmap,rs1[i]|64));
2987   }
2988   else
2989   {
2990     assert(opcode2[i]==0x10);
2991     //if((source[i]&0x3f)==0x10) // RFE
2992     {
2993       emit_readword(&Status,0);
2994       emit_andimm(0,0x3c,1);
2995       emit_andimm(0,~0xf,0);
2996       emit_orrshr_imm(1,2,0);
2997       emit_writeword(0,&Status);
2998     }
2999   }
3000 }
3001
3002 static void cop1_unusable(int i,struct regstat *i_regs)
3003 {
3004   // XXX: should just just do the exception instead
3005   //if(!cop1_usable)
3006   {
3007     void *jaddr=out;
3008     emit_jmp(0);
3009     add_stub_r(FP_STUB,jaddr,out,i,0,i_regs,is_delayslot,0);
3010   }
3011 }
3012
3013 static void cop1_assemble(int i,struct regstat *i_regs)
3014 {
3015   cop1_unusable(i, i_regs);
3016 }
3017
3018 static void c1ls_assemble(int i,struct regstat *i_regs)
3019 {
3020   cop1_unusable(i, i_regs);
3021 }
3022
3023 // FP_STUB
3024 static void do_cop1stub(int n)
3025 {
3026   literal_pool(256);
3027   assem_debug("do_cop1stub %x\n",start+stubs[n].a*4);
3028   set_jump_target(stubs[n].addr, out);
3029   int i=stubs[n].a;
3030 //  int rs=stubs[n].b;
3031   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3032   int ds=stubs[n].d;
3033   if(!ds) {
3034     load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
3035     //if(i_regs!=&regs[i]) printf("oops: regs[i]=%x i_regs=%x",(int)&regs[i],(int)i_regs);
3036   }
3037   //else {printf("fp exception in delay slot\n");}
3038   wb_dirtys(i_regs->regmap_entry,i_regs->wasdirty);
3039   if(regs[i].regmap_entry[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
3040   emit_movimm(start+(i-ds)*4,EAX); // Get PC
3041   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3042   emit_jmp(ds?fp_exception_ds:fp_exception);
3043 }
3044
3045 static void cop2_get_dreg(u_int copr,signed char tl,signed char temp)
3046 {
3047   switch (copr) {
3048     case 1:
3049     case 3:
3050     case 5:
3051     case 8:
3052     case 9:
3053     case 10:
3054     case 11:
3055       emit_readword(&reg_cop2d[copr],tl);
3056       emit_signextend16(tl,tl);
3057       emit_writeword(tl,&reg_cop2d[copr]); // hmh
3058       break;
3059     case 7:
3060     case 16:
3061     case 17:
3062     case 18:
3063     case 19:
3064       emit_readword(&reg_cop2d[copr],tl);
3065       emit_andimm(tl,0xffff,tl);
3066       emit_writeword(tl,&reg_cop2d[copr]);
3067       break;
3068     case 15:
3069       emit_readword(&reg_cop2d[14],tl); // SXY2
3070       emit_writeword(tl,&reg_cop2d[copr]);
3071       break;
3072     case 28:
3073     case 29:
3074       emit_readword(&reg_cop2d[9],temp);
3075       emit_testimm(temp,0x8000); // do we need this?
3076       emit_andimm(temp,0xf80,temp);
3077       emit_andne_imm(temp,0,temp);
3078       emit_shrimm(temp,7,tl);
3079       emit_readword(&reg_cop2d[10],temp);
3080       emit_testimm(temp,0x8000);
3081       emit_andimm(temp,0xf80,temp);
3082       emit_andne_imm(temp,0,temp);
3083       emit_orrshr_imm(temp,2,tl);
3084       emit_readword(&reg_cop2d[11],temp);
3085       emit_testimm(temp,0x8000);
3086       emit_andimm(temp,0xf80,temp);
3087       emit_andne_imm(temp,0,temp);
3088       emit_orrshl_imm(temp,3,tl);
3089       emit_writeword(tl,&reg_cop2d[copr]);
3090       break;
3091     default:
3092       emit_readword(&reg_cop2d[copr],tl);
3093       break;
3094   }
3095 }
3096
3097 static void cop2_put_dreg(u_int copr,signed char sl,signed char temp)
3098 {
3099   switch (copr) {
3100     case 15:
3101       emit_readword(&reg_cop2d[13],temp);  // SXY1
3102       emit_writeword(sl,&reg_cop2d[copr]);
3103       emit_writeword(temp,&reg_cop2d[12]); // SXY0
3104       emit_readword(&reg_cop2d[14],temp);  // SXY2
3105       emit_writeword(sl,&reg_cop2d[14]);
3106       emit_writeword(temp,&reg_cop2d[13]); // SXY1
3107       break;
3108     case 28:
3109       emit_andimm(sl,0x001f,temp);
3110       emit_shlimm(temp,7,temp);
3111       emit_writeword(temp,&reg_cop2d[9]);
3112       emit_andimm(sl,0x03e0,temp);
3113       emit_shlimm(temp,2,temp);
3114       emit_writeword(temp,&reg_cop2d[10]);
3115       emit_andimm(sl,0x7c00,temp);
3116       emit_shrimm(temp,3,temp);
3117       emit_writeword(temp,&reg_cop2d[11]);
3118       emit_writeword(sl,&reg_cop2d[28]);
3119       break;
3120     case 30:
3121       emit_movs(sl,temp);
3122       emit_mvnmi(temp,temp);
3123 #if defined(HAVE_ARMV5) || defined(__aarch64__)
3124       emit_clz(temp,temp);
3125 #else
3126       emit_movs(temp,HOST_TEMPREG);
3127       emit_movimm(0,temp);
3128       emit_jeq((int)out+4*4);
3129       emit_addpl_imm(temp,1,temp);
3130       emit_lslpls_imm(HOST_TEMPREG,1,HOST_TEMPREG);
3131       emit_jns((int)out-2*4);
3132 #endif
3133       emit_writeword(sl,&reg_cop2d[30]);
3134       emit_writeword(temp,&reg_cop2d[31]);
3135       break;
3136     case 31:
3137       break;
3138     default:
3139       emit_writeword(sl,&reg_cop2d[copr]);
3140       break;
3141   }
3142 }
3143
3144 static void c2ls_assemble(int i,struct regstat *i_regs)
3145 {
3146   int s,tl;
3147   int ar;
3148   int offset;
3149   int memtarget=0,c=0;
3150   void *jaddr2=NULL;
3151   enum stub_type type;
3152   int agr=AGEN1+(i&1);
3153   int fastio_reg_override=0;
3154   u_int hr,reglist=0;
3155   u_int copr=(source[i]>>16)&0x1f;
3156   s=get_reg(i_regs->regmap,rs1[i]);
3157   tl=get_reg(i_regs->regmap,FTEMP);
3158   offset=imm[i];
3159   assert(rs1[i]>0);
3160   assert(tl>=0);
3161
3162   for(hr=0;hr<HOST_REGS;hr++) {
3163     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3164   }
3165   if(i_regs->regmap[HOST_CCREG]==CCREG)
3166     reglist&=~(1<<HOST_CCREG);
3167
3168   // get the address
3169   if (opcode[i]==0x3a) { // SWC2
3170     ar=get_reg(i_regs->regmap,agr);
3171     if(ar<0) ar=get_reg(i_regs->regmap,-1);
3172     reglist|=1<<ar;
3173   } else { // LWC2
3174     ar=tl;
3175   }
3176   if(s>=0) c=(i_regs->wasconst>>s)&1;
3177   memtarget=c&&(((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE);
3178   if (!offset&&!c&&s>=0) ar=s;
3179   assert(ar>=0);
3180
3181   if (opcode[i]==0x3a) { // SWC2
3182     cop2_get_dreg(copr,tl,HOST_TEMPREG);
3183     type=STOREW_STUB;
3184   }
3185   else
3186     type=LOADW_STUB;
3187
3188   if(c&&!memtarget) {
3189     jaddr2=out;
3190     emit_jmp(0); // inline_readstub/inline_writestub?
3191   }
3192   else {
3193     if(!c) {
3194       jaddr2=emit_fastpath_cmp_jump(i,ar,&fastio_reg_override);
3195     }
3196     else if(ram_offset&&memtarget) {
3197       emit_addimm(ar,ram_offset,HOST_TEMPREG);
3198       fastio_reg_override=HOST_TEMPREG;
3199     }
3200     if (opcode[i]==0x32) { // LWC2
3201       int a=ar;
3202       if(fastio_reg_override) a=fastio_reg_override;
3203       emit_readword_indexed(0,a,tl);
3204     }
3205     if (opcode[i]==0x3a) { // SWC2
3206       #ifdef DESTRUCTIVE_SHIFT
3207       if(!offset&&!c&&s>=0) emit_mov(s,ar);
3208       #endif
3209       int a=ar;
3210       if(fastio_reg_override) a=fastio_reg_override;
3211       emit_writeword_indexed(tl,0,a);
3212     }
3213   }
3214   if(jaddr2)
3215     add_stub_r(type,jaddr2,out,i,ar,i_regs,ccadj[i],reglist);
3216   if(opcode[i]==0x3a) // SWC2
3217   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
3218 #if defined(HOST_IMM8)
3219     int ir=get_reg(i_regs->regmap,INVCP);
3220     assert(ir>=0);
3221     emit_cmpmem_indexedsr12_reg(ir,ar,1);
3222 #else
3223     emit_cmpmem_indexedsr12_imm(invalid_code,ar,1);
3224 #endif
3225     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3226     emit_callne(invalidate_addr_reg[ar]);
3227     #else
3228     void *jaddr3 = out;
3229     emit_jne(0);
3230     add_stub(INVCODE_STUB,jaddr3,out,reglist|(1<<HOST_CCREG),ar,0,0,0);
3231     #endif
3232   }
3233   if (opcode[i]==0x32) { // LWC2
3234     cop2_put_dreg(copr,tl,HOST_TEMPREG);
3235   }
3236 }
3237
3238 static void cop2_assemble(int i,struct regstat *i_regs)
3239 {
3240   u_int copr=(source[i]>>11)&0x1f;
3241   signed char temp=get_reg(i_regs->regmap,-1);
3242   if (opcode2[i]==0) { // MFC2
3243     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3244     if(tl>=0&&rt1[i]!=0)
3245       cop2_get_dreg(copr,tl,temp);
3246   }
3247   else if (opcode2[i]==4) { // MTC2
3248     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3249     cop2_put_dreg(copr,sl,temp);
3250   }
3251   else if (opcode2[i]==2) // CFC2
3252   {
3253     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3254     if(tl>=0&&rt1[i]!=0)
3255       emit_readword(&reg_cop2c[copr],tl);
3256   }
3257   else if (opcode2[i]==6) // CTC2
3258   {
3259     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3260     switch(copr) {
3261       case 4:
3262       case 12:
3263       case 20:
3264       case 26:
3265       case 27:
3266       case 29:
3267       case 30:
3268         emit_signextend16(sl,temp);
3269         break;
3270       case 31:
3271         //value = value & 0x7ffff000;
3272         //if (value & 0x7f87e000) value |= 0x80000000;
3273         emit_shrimm(sl,12,temp);
3274         emit_shlimm(temp,12,temp);
3275         emit_testimm(temp,0x7f000000);
3276         emit_testeqimm(temp,0x00870000);
3277         emit_testeqimm(temp,0x0000e000);
3278         emit_orrne_imm(temp,0x80000000,temp);
3279         break;
3280       default:
3281         temp=sl;
3282         break;
3283     }
3284     emit_writeword(temp,&reg_cop2c[copr]);
3285     assert(sl>=0);
3286   }
3287 }
3288
3289 #ifndef multdiv_assemble
3290 void multdiv_assemble(int i,struct regstat *i_regs)
3291 {
3292   printf("Need multdiv_assemble for this architecture.\n");
3293   exit(1);
3294 }
3295 #endif
3296
3297 void mov_assemble(int i,struct regstat *i_regs)
3298 {
3299   //if(opcode2[i]==0x10||opcode2[i]==0x12) { // MFHI/MFLO
3300   //if(opcode2[i]==0x11||opcode2[i]==0x13) { // MTHI/MTLO
3301   if(rt1[i]) {
3302     signed char sh,sl,th,tl;
3303     th=get_reg(i_regs->regmap,rt1[i]|64);
3304     tl=get_reg(i_regs->regmap,rt1[i]);
3305     //assert(tl>=0);
3306     if(tl>=0) {
3307       sh=get_reg(i_regs->regmap,rs1[i]|64);
3308       sl=get_reg(i_regs->regmap,rs1[i]);
3309       if(sl>=0) emit_mov(sl,tl);
3310       else emit_loadreg(rs1[i],tl);
3311       if(th>=0) {
3312         if(sh>=0) emit_mov(sh,th);
3313         else emit_loadreg(rs1[i]|64,th);
3314       }
3315     }
3316   }
3317 }
3318
3319 void syscall_assemble(int i,struct regstat *i_regs)
3320 {
3321   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3322   assert(ccreg==HOST_CCREG);
3323   assert(!is_delayslot);
3324   (void)ccreg;
3325   emit_movimm(start+i*4,EAX); // Get PC
3326   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3327   emit_jmp(jump_syscall_hle); // XXX
3328 }
3329
3330 void hlecall_assemble(int i,struct regstat *i_regs)
3331 {
3332   extern void psxNULL();
3333   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3334   assert(ccreg==HOST_CCREG);
3335   assert(!is_delayslot);
3336   (void)ccreg;
3337   emit_movimm(start+i*4+4,0); // Get PC
3338   uint32_t hleCode = source[i] & 0x03ffffff;
3339   if (hleCode >= ARRAY_SIZE(psxHLEt))
3340     emit_movimm((uintptr_t)psxNULL,1);
3341   else
3342     emit_movimm((uintptr_t)psxHLEt[hleCode],1);
3343   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // XXX
3344   emit_jmp(jump_hlecall);
3345 }
3346
3347 void intcall_assemble(int i,struct regstat *i_regs)
3348 {
3349   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3350   assert(ccreg==HOST_CCREG);
3351   assert(!is_delayslot);
3352   (void)ccreg;
3353   emit_movimm(start+i*4,0); // Get PC
3354   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3355   emit_jmp(jump_intcall);
3356 }
3357
3358 static void speculate_mov(int rs,int rt)
3359 {
3360   if(rt!=0) {
3361     smrv_strong_next|=1<<rt;
3362     smrv[rt]=smrv[rs];
3363   }
3364 }
3365
3366 static void speculate_mov_weak(int rs,int rt)
3367 {
3368   if(rt!=0) {
3369     smrv_weak_next|=1<<rt;
3370     smrv[rt]=smrv[rs];
3371   }
3372 }
3373
3374 static void speculate_register_values(int i)
3375 {
3376   if(i==0) {
3377     memcpy(smrv,psxRegs.GPR.r,sizeof(smrv));
3378     // gp,sp are likely to stay the same throughout the block
3379     smrv_strong_next=(1<<28)|(1<<29)|(1<<30);
3380     smrv_weak_next=~smrv_strong_next;
3381     //printf(" llr %08x\n", smrv[4]);
3382   }
3383   smrv_strong=smrv_strong_next;
3384   smrv_weak=smrv_weak_next;
3385   switch(itype[i]) {
3386     case ALU:
3387       if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3388       else if((smrv_strong>>rs2[i])&1) speculate_mov(rs2[i],rt1[i]);
3389       else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3390       else if((smrv_weak>>rs2[i])&1) speculate_mov_weak(rs2[i],rt1[i]);
3391       else {
3392         smrv_strong_next&=~(1<<rt1[i]);
3393         smrv_weak_next&=~(1<<rt1[i]);
3394       }
3395       break;
3396     case SHIFTIMM:
3397       smrv_strong_next&=~(1<<rt1[i]);
3398       smrv_weak_next&=~(1<<rt1[i]);
3399       // fallthrough
3400     case IMM16:
3401       if(rt1[i]&&is_const(&regs[i],rt1[i])) {
3402         int value,hr=get_reg(regs[i].regmap,rt1[i]);
3403         if(hr>=0) {
3404           if(get_final_value(hr,i,&value))
3405                smrv[rt1[i]]=value;
3406           else smrv[rt1[i]]=constmap[i][hr];
3407           smrv_strong_next|=1<<rt1[i];
3408         }
3409       }
3410       else {
3411         if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3412         else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3413       }
3414       break;
3415     case LOAD:
3416       if(start<0x2000&&(rt1[i]==26||(smrv[rt1[i]]>>24)==0xa0)) {
3417         // special case for BIOS
3418         smrv[rt1[i]]=0xa0000000;
3419         smrv_strong_next|=1<<rt1[i];
3420         break;
3421       }
3422       // fallthrough
3423     case SHIFT:
3424     case LOADLR:
3425     case MOV:
3426       smrv_strong_next&=~(1<<rt1[i]);
3427       smrv_weak_next&=~(1<<rt1[i]);
3428       break;
3429     case COP0:
3430     case COP2:
3431       if(opcode2[i]==0||opcode2[i]==2) { // MFC/CFC
3432         smrv_strong_next&=~(1<<rt1[i]);
3433         smrv_weak_next&=~(1<<rt1[i]);
3434       }
3435       break;
3436     case C2LS:
3437       if (opcode[i]==0x32) { // LWC2
3438         smrv_strong_next&=~(1<<rt1[i]);
3439         smrv_weak_next&=~(1<<rt1[i]);
3440       }
3441       break;
3442   }
3443 #if 0
3444   int r=4;
3445   printf("x %08x %08x %d %d c %08x %08x\n",smrv[r],start+i*4,
3446     ((smrv_strong>>r)&1),(smrv_weak>>r)&1,regs[i].isconst,regs[i].wasconst);
3447 #endif
3448 }
3449
3450 void ds_assemble(int i,struct regstat *i_regs)
3451 {
3452   speculate_register_values(i);
3453   is_delayslot=1;
3454   switch(itype[i]) {
3455     case ALU:
3456       alu_assemble(i,i_regs);break;
3457     case IMM16:
3458       imm16_assemble(i,i_regs);break;
3459     case SHIFT:
3460       shift_assemble(i,i_regs);break;
3461     case SHIFTIMM:
3462       shiftimm_assemble(i,i_regs);break;
3463     case LOAD:
3464       load_assemble(i,i_regs);break;
3465     case LOADLR:
3466       loadlr_assemble(i,i_regs);break;
3467     case STORE:
3468       store_assemble(i,i_regs);break;
3469     case STORELR:
3470       storelr_assemble(i,i_regs);break;
3471     case COP0:
3472       cop0_assemble(i,i_regs);break;
3473     case COP1:
3474       cop1_assemble(i,i_regs);break;
3475     case C1LS:
3476       c1ls_assemble(i,i_regs);break;
3477     case COP2:
3478       cop2_assemble(i,i_regs);break;
3479     case C2LS:
3480       c2ls_assemble(i,i_regs);break;
3481     case C2OP:
3482       c2op_assemble(i,i_regs);break;
3483     case MULTDIV:
3484       multdiv_assemble(i,i_regs);break;
3485     case MOV:
3486       mov_assemble(i,i_regs);break;
3487     case SYSCALL:
3488     case HLECALL:
3489     case INTCALL:
3490     case SPAN:
3491     case UJUMP:
3492     case RJUMP:
3493     case CJUMP:
3494     case SJUMP:
3495       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
3496   }
3497   is_delayslot=0;
3498 }
3499
3500 // Is the branch target a valid internal jump?
3501 static int internal_branch(int addr)
3502 {
3503   if(addr&1) return 0; // Indirect (register) jump
3504   if(addr>=start && addr<start+slen*4-4)
3505   {
3506     return 1;
3507   }
3508   return 0;
3509 }
3510
3511 static void wb_invalidate(signed char pre[],signed char entry[],uint64_t dirty,uint64_t u)
3512 {
3513   int hr;
3514   for(hr=0;hr<HOST_REGS;hr++) {
3515     if(hr!=EXCLUDE_REG) {
3516       if(pre[hr]!=entry[hr]) {
3517         if(pre[hr]>=0) {
3518           if((dirty>>hr)&1) {
3519             if(get_reg(entry,pre[hr])<0) {
3520               assert(pre[hr]<64);
3521               if(!((u>>pre[hr])&1))
3522                 emit_storereg(pre[hr],hr);
3523             }
3524           }
3525         }
3526       }
3527     }
3528   }
3529   // Move from one register to another (no writeback)
3530   for(hr=0;hr<HOST_REGS;hr++) {
3531     if(hr!=EXCLUDE_REG) {
3532       if(pre[hr]!=entry[hr]) {
3533         if(pre[hr]>=0&&(pre[hr]&63)<TEMPREG) {
3534           int nr;
3535           if((nr=get_reg(entry,pre[hr]))>=0) {
3536             emit_mov(hr,nr);
3537           }
3538         }
3539       }
3540     }
3541   }
3542 }
3543
3544 // Load the specified registers
3545 // This only loads the registers given as arguments because
3546 // we don't want to load things that will be overwritten
3547 static void load_regs(signed char entry[],signed char regmap[],int rs1,int rs2)
3548 {
3549   int hr;
3550   // Load 32-bit regs
3551   for(hr=0;hr<HOST_REGS;hr++) {
3552     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3553       if(entry[hr]!=regmap[hr]) {
3554         if(regmap[hr]==rs1||regmap[hr]==rs2)
3555         {
3556           if(regmap[hr]==0) {
3557             emit_zeroreg(hr);
3558           }
3559           else
3560           {
3561             emit_loadreg(regmap[hr],hr);
3562           }
3563         }
3564       }
3565     }
3566   }
3567 }
3568
3569 // Load registers prior to the start of a loop
3570 // so that they are not loaded within the loop
3571 static void loop_preload(signed char pre[],signed char entry[])
3572 {
3573   int hr;
3574   for(hr=0;hr<HOST_REGS;hr++) {
3575     if(hr!=EXCLUDE_REG) {
3576       if(pre[hr]!=entry[hr]) {
3577         if(entry[hr]>=0) {
3578           if(get_reg(pre,entry[hr])<0) {
3579             assem_debug("loop preload:\n");
3580             //printf("loop preload: %d\n",hr);
3581             if(entry[hr]==0) {
3582               emit_zeroreg(hr);
3583             }
3584             else if(entry[hr]<TEMPREG)
3585             {
3586               emit_loadreg(entry[hr],hr);
3587             }
3588             else if(entry[hr]-64<TEMPREG)
3589             {
3590               emit_loadreg(entry[hr],hr);
3591             }
3592           }
3593         }
3594       }
3595     }
3596   }
3597 }
3598
3599 // Generate address for load/store instruction
3600 // goes to AGEN for writes, FTEMP for LOADLR and cop1/2 loads
3601 void address_generation(int i,struct regstat *i_regs,signed char entry[])
3602 {
3603   if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS||itype[i]==C2LS) {
3604     int ra=-1;
3605     int agr=AGEN1+(i&1);
3606     if(itype[i]==LOAD) {
3607       ra=get_reg(i_regs->regmap,rt1[i]);
3608       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3609       assert(ra>=0);
3610     }
3611     if(itype[i]==LOADLR) {
3612       ra=get_reg(i_regs->regmap,FTEMP);
3613     }
3614     if(itype[i]==STORE||itype[i]==STORELR) {
3615       ra=get_reg(i_regs->regmap,agr);
3616       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3617     }
3618     if(itype[i]==C1LS||itype[i]==C2LS) {
3619       if ((opcode[i]&0x3b)==0x31||(opcode[i]&0x3b)==0x32) // LWC1/LDC1/LWC2/LDC2
3620         ra=get_reg(i_regs->regmap,FTEMP);
3621       else { // SWC1/SDC1/SWC2/SDC2
3622         ra=get_reg(i_regs->regmap,agr);
3623         if(ra<0) ra=get_reg(i_regs->regmap,-1);
3624       }
3625     }
3626     int rs=get_reg(i_regs->regmap,rs1[i]);
3627     if(ra>=0) {
3628       int offset=imm[i];
3629       int c=(i_regs->wasconst>>rs)&1;
3630       if(rs1[i]==0) {
3631         // Using r0 as a base address
3632         if(!entry||entry[ra]!=agr) {
3633           if (opcode[i]==0x22||opcode[i]==0x26) {
3634             emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3635           }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3636             emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3637           }else{
3638             emit_movimm(offset,ra);
3639           }
3640         } // else did it in the previous cycle
3641       }
3642       else if(rs<0) {
3643         if(!entry||entry[ra]!=rs1[i])
3644           emit_loadreg(rs1[i],ra);
3645         //if(!entry||entry[ra]!=rs1[i])
3646         //  printf("poor load scheduling!\n");
3647       }
3648       else if(c) {
3649         if(rs1[i]!=rt1[i]||itype[i]!=LOAD) {
3650           if(!entry||entry[ra]!=agr) {
3651             if (opcode[i]==0x22||opcode[i]==0x26) {
3652               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3653             }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3654               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3655             }else{
3656               emit_movimm(constmap[i][rs]+offset,ra);
3657               regs[i].loadedconst|=1<<ra;
3658             }
3659           } // else did it in the previous cycle
3660         } // else load_consts already did it
3661       }
3662       if(offset&&!c&&rs1[i]) {
3663         if(rs>=0) {
3664           emit_addimm(rs,offset,ra);
3665         }else{
3666           emit_addimm(ra,offset,ra);
3667         }
3668       }
3669     }
3670   }
3671   // Preload constants for next instruction
3672   if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS||itype[i+1]==C2LS) {
3673     int agr,ra;
3674     // Actual address
3675     agr=AGEN1+((i+1)&1);
3676     ra=get_reg(i_regs->regmap,agr);
3677     if(ra>=0) {
3678       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
3679       int offset=imm[i+1];
3680       int c=(regs[i+1].wasconst>>rs)&1;
3681       if(c&&(rs1[i+1]!=rt1[i+1]||itype[i+1]!=LOAD)) {
3682         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3683           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3684         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3685           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3686         }else{
3687           emit_movimm(constmap[i+1][rs]+offset,ra);
3688           regs[i+1].loadedconst|=1<<ra;
3689         }
3690       }
3691       else if(rs1[i+1]==0) {
3692         // Using r0 as a base address
3693         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3694           emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3695         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3696           emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3697         }else{
3698           emit_movimm(offset,ra);
3699         }
3700       }
3701     }
3702   }
3703 }
3704
3705 static int get_final_value(int hr, int i, int *value)
3706 {
3707   int reg=regs[i].regmap[hr];
3708   while(i<slen-1) {
3709     if(regs[i+1].regmap[hr]!=reg) break;
3710     if(!((regs[i+1].isconst>>hr)&1)) break;
3711     if(bt[i+1]) break;
3712     i++;
3713   }
3714   if(i<slen-1) {
3715     if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
3716       *value=constmap[i][hr];
3717       return 1;
3718     }
3719     if(!bt[i+1]) {
3720       if(itype[i+1]==UJUMP||itype[i+1]==RJUMP||itype[i+1]==CJUMP||itype[i+1]==SJUMP) {
3721         // Load in delay slot, out-of-order execution
3722         if(itype[i+2]==LOAD&&rs1[i+2]==reg&&rt1[i+2]==reg&&((regs[i+1].wasconst>>hr)&1))
3723         {
3724           // Precompute load address
3725           *value=constmap[i][hr]+imm[i+2];
3726           return 1;
3727         }
3728       }
3729       if(itype[i+1]==LOAD&&rs1[i+1]==reg&&rt1[i+1]==reg)
3730       {
3731         // Precompute load address
3732         *value=constmap[i][hr]+imm[i+1];
3733         //printf("c=%x imm=%lx\n",(long)constmap[i][hr],imm[i+1]);
3734         return 1;
3735       }
3736     }
3737   }
3738   *value=constmap[i][hr];
3739   //printf("c=%lx\n",(long)constmap[i][hr]);
3740   if(i==slen-1) return 1;
3741   assert(reg < 64);
3742   return !((unneeded_reg[i+1]>>reg)&1);
3743 }
3744
3745 // Load registers with known constants
3746 static void load_consts(signed char pre[],signed char regmap[],int i)
3747 {
3748   int hr,hr2;
3749   // propagate loaded constant flags
3750   if(i==0||bt[i])
3751     regs[i].loadedconst=0;
3752   else {
3753     for(hr=0;hr<HOST_REGS;hr++) {
3754       if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((regs[i-1].isconst>>hr)&1)&&pre[hr]==regmap[hr]
3755          &&regmap[hr]==regs[i-1].regmap[hr]&&((regs[i-1].loadedconst>>hr)&1))
3756       {
3757         regs[i].loadedconst|=1<<hr;
3758       }
3759     }
3760   }
3761   // Load 32-bit regs
3762   for(hr=0;hr<HOST_REGS;hr++) {
3763     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3764       //if(entry[hr]!=regmap[hr]) {
3765       if(!((regs[i].loadedconst>>hr)&1)) {
3766         assert(regmap[hr]<64);
3767         if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
3768           int value,similar=0;
3769           if(get_final_value(hr,i,&value)) {
3770             // see if some other register has similar value
3771             for(hr2=0;hr2<HOST_REGS;hr2++) {
3772               if(hr2!=EXCLUDE_REG&&((regs[i].loadedconst>>hr2)&1)) {
3773                 if(is_similar_value(value,constmap[i][hr2])) {
3774                   similar=1;
3775                   break;
3776                 }
3777               }
3778             }
3779             if(similar) {
3780               int value2;
3781               if(get_final_value(hr2,i,&value2)) // is this needed?
3782                 emit_movimm_from(value2,hr2,value,hr);
3783               else
3784                 emit_movimm(value,hr);
3785             }
3786             else if(value==0) {
3787               emit_zeroreg(hr);
3788             }
3789             else {
3790               emit_movimm(value,hr);
3791             }
3792           }
3793           regs[i].loadedconst|=1<<hr;
3794         }
3795       }
3796     }
3797   }
3798 }
3799
3800 void load_all_consts(signed char regmap[], u_int dirty, int i)
3801 {
3802   int hr;
3803   // Load 32-bit regs
3804   for(hr=0;hr<HOST_REGS;hr++) {
3805     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
3806       assert(regmap[hr] < 64);
3807       if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
3808         int value=constmap[i][hr];
3809         if(value==0) {
3810           emit_zeroreg(hr);
3811         }
3812         else {
3813           emit_movimm(value,hr);
3814         }
3815       }
3816     }
3817   }
3818 }
3819
3820 // Write out all dirty registers (except cycle count)
3821 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty)
3822 {
3823   int hr;
3824   for(hr=0;hr<HOST_REGS;hr++) {
3825     if(hr!=EXCLUDE_REG) {
3826       if(i_regmap[hr]>0) {
3827         if(i_regmap[hr]!=CCREG) {
3828           if((i_dirty>>hr)&1) {
3829             assert(i_regmap[hr]<64);
3830             emit_storereg(i_regmap[hr],hr);
3831           }
3832         }
3833       }
3834     }
3835   }
3836 }
3837
3838 // Write out dirty registers that we need to reload (pair with load_needed_regs)
3839 // This writes the registers not written by store_regs_bt
3840 void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr)
3841 {
3842   int hr;
3843   int t=(addr-start)>>2;
3844   for(hr=0;hr<HOST_REGS;hr++) {
3845     if(hr!=EXCLUDE_REG) {
3846       if(i_regmap[hr]>0) {
3847         if(i_regmap[hr]!=CCREG) {
3848           if(i_regmap[hr]==regs[t].regmap_entry[hr] && ((regs[t].dirty>>hr)&1)) {
3849             if((i_dirty>>hr)&1) {
3850               assert(i_regmap[hr]<64);
3851               emit_storereg(i_regmap[hr],hr);
3852             }
3853           }
3854         }
3855       }
3856     }
3857   }
3858 }
3859
3860 // Load all registers (except cycle count)
3861 void load_all_regs(signed char i_regmap[])
3862 {
3863   int hr;
3864   for(hr=0;hr<HOST_REGS;hr++) {
3865     if(hr!=EXCLUDE_REG) {
3866       if(i_regmap[hr]==0) {
3867         emit_zeroreg(hr);
3868       }
3869       else
3870       if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
3871       {
3872         emit_loadreg(i_regmap[hr],hr);
3873       }
3874     }
3875   }
3876 }
3877
3878 // Load all current registers also needed by next instruction
3879 void load_needed_regs(signed char i_regmap[],signed char next_regmap[])
3880 {
3881   int hr;
3882   for(hr=0;hr<HOST_REGS;hr++) {
3883     if(hr!=EXCLUDE_REG) {
3884       if(get_reg(next_regmap,i_regmap[hr])>=0) {
3885         if(i_regmap[hr]==0) {
3886           emit_zeroreg(hr);
3887         }
3888         else
3889         if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
3890         {
3891           emit_loadreg(i_regmap[hr],hr);
3892         }
3893       }
3894     }
3895   }
3896 }
3897
3898 // Load all regs, storing cycle count if necessary
3899 void load_regs_entry(int t)
3900 {
3901   int hr;
3902   if(is_ds[t]) emit_addimm(HOST_CCREG,CLOCK_ADJUST(1),HOST_CCREG);
3903   else if(ccadj[t]) emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[t]),HOST_CCREG);
3904   if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
3905     emit_storereg(CCREG,HOST_CCREG);
3906   }
3907   // Load 32-bit regs
3908   for(hr=0;hr<HOST_REGS;hr++) {
3909     if(regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
3910       if(regs[t].regmap_entry[hr]==0) {
3911         emit_zeroreg(hr);
3912       }
3913       else if(regs[t].regmap_entry[hr]!=CCREG)
3914       {
3915         emit_loadreg(regs[t].regmap_entry[hr],hr);
3916       }
3917     }
3918   }
3919 }
3920
3921 // Store dirty registers prior to branch
3922 void store_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3923 {
3924   if(internal_branch(addr))
3925   {
3926     int t=(addr-start)>>2;
3927     int hr;
3928     for(hr=0;hr<HOST_REGS;hr++) {
3929       if(hr!=EXCLUDE_REG) {
3930         if(i_regmap[hr]>0 && i_regmap[hr]!=CCREG) {
3931           if(i_regmap[hr]!=regs[t].regmap_entry[hr] || !((regs[t].dirty>>hr)&1)) {
3932             if((i_dirty>>hr)&1) {
3933               assert(i_regmap[hr]<64);
3934               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
3935                 emit_storereg(i_regmap[hr],hr);
3936             }
3937           }
3938         }
3939       }
3940     }
3941   }
3942   else
3943   {
3944     // Branch out of this block, write out all dirty regs
3945     wb_dirtys(i_regmap,i_dirty);
3946   }
3947 }
3948
3949 // Load all needed registers for branch target
3950 static void load_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3951 {
3952   //if(addr>=start && addr<(start+slen*4))
3953   if(internal_branch(addr))
3954   {
3955     int t=(addr-start)>>2;
3956     int hr;
3957     // Store the cycle count before loading something else
3958     if(i_regmap[HOST_CCREG]!=CCREG) {
3959       assert(i_regmap[HOST_CCREG]==-1);
3960     }
3961     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
3962       emit_storereg(CCREG,HOST_CCREG);
3963     }
3964     // Load 32-bit regs
3965     for(hr=0;hr<HOST_REGS;hr++) {
3966       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
3967         if(i_regmap[hr]!=regs[t].regmap_entry[hr]) {
3968           if(regs[t].regmap_entry[hr]==0) {
3969             emit_zeroreg(hr);
3970           }
3971           else if(regs[t].regmap_entry[hr]!=CCREG)
3972           {
3973             emit_loadreg(regs[t].regmap_entry[hr],hr);
3974           }
3975         }
3976       }
3977     }
3978   }
3979 }
3980
3981 static int match_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3982 {
3983   if(addr>=start && addr<start+slen*4-4)
3984   {
3985     int t=(addr-start)>>2;
3986     int hr;
3987     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) return 0;
3988     for(hr=0;hr<HOST_REGS;hr++)
3989     {
3990       if(hr!=EXCLUDE_REG)
3991       {
3992         if(i_regmap[hr]!=regs[t].regmap_entry[hr])
3993         {
3994           if(regs[t].regmap_entry[hr]>=0&&(regs[t].regmap_entry[hr]|64)<TEMPREG+64)
3995           {
3996             return 0;
3997           }
3998           else
3999           if((i_dirty>>hr)&1)
4000           {
4001             if(i_regmap[hr]<TEMPREG)
4002             {
4003               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4004                 return 0;
4005             }
4006             else if(i_regmap[hr]>=64&&i_regmap[hr]<TEMPREG+64)
4007             {
4008               assert(0);
4009             }
4010           }
4011         }
4012         else // Same register but is it 32-bit or dirty?
4013         if(i_regmap[hr]>=0)
4014         {
4015           if(!((regs[t].dirty>>hr)&1))
4016           {
4017             if((i_dirty>>hr)&1)
4018             {
4019               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4020               {
4021                 //printf("%x: dirty no match\n",addr);
4022                 return 0;
4023               }
4024             }
4025           }
4026         }
4027       }
4028     }
4029     // Delay slots are not valid branch targets
4030     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP)) return 0;
4031     // Delay slots require additional processing, so do not match
4032     if(is_ds[t]) return 0;
4033   }
4034   else
4035   {
4036     int hr;
4037     for(hr=0;hr<HOST_REGS;hr++)
4038     {
4039       if(hr!=EXCLUDE_REG)
4040       {
4041         if(i_regmap[hr]>=0)
4042         {
4043           if(hr!=HOST_CCREG||i_regmap[hr]!=CCREG)
4044           {
4045             if((i_dirty>>hr)&1)
4046             {
4047               return 0;
4048             }
4049           }
4050         }
4051       }
4052     }
4053   }
4054   return 1;
4055 }
4056
4057 #ifdef DRC_DBG
4058 static void drc_dbg_emit_do_cmp(int i)
4059 {
4060   extern void do_insn_cmp();
4061   extern int cycle;
4062   u_int hr,reglist=0;
4063
4064   for(hr=0;hr<HOST_REGS;hr++)
4065     if(regs[i].regmap[hr]>=0) reglist|=1<<hr;
4066   save_regs(reglist);
4067   emit_movimm(start+i*4,0);
4068   emit_writeword(0,&pcaddr);
4069   emit_call(do_insn_cmp);
4070   //emit_readword(&cycle,0);
4071   //emit_addimm(0,2,0);
4072   //emit_writeword(0,&cycle);
4073   restore_regs(reglist);
4074 }
4075 #else
4076 #define drc_dbg_emit_do_cmp(x)
4077 #endif
4078
4079 // Used when a branch jumps into the delay slot of another branch
4080 void ds_assemble_entry(int i)
4081 {
4082   int t=(ba[i]-start)>>2;
4083   if (!instr_addr[t])
4084     instr_addr[t] = out;
4085   assem_debug("Assemble delay slot at %x\n",ba[i]);
4086   assem_debug("<->\n");
4087   drc_dbg_emit_do_cmp(t);
4088   if(regs[t].regmap_entry[HOST_CCREG]==CCREG&&regs[t].regmap[HOST_CCREG]!=CCREG)
4089     wb_register(CCREG,regs[t].regmap_entry,regs[t].wasdirty);
4090   load_regs(regs[t].regmap_entry,regs[t].regmap,rs1[t],rs2[t]);
4091   address_generation(t,&regs[t],regs[t].regmap_entry);
4092   if(itype[t]==STORE||itype[t]==STORELR||(opcode[t]&0x3b)==0x39||(opcode[t]&0x3b)==0x3a)
4093     load_regs(regs[t].regmap_entry,regs[t].regmap,INVCP,INVCP);
4094   is_delayslot=0;
4095   switch(itype[t]) {
4096     case ALU:
4097       alu_assemble(t,&regs[t]);break;
4098     case IMM16:
4099       imm16_assemble(t,&regs[t]);break;
4100     case SHIFT:
4101       shift_assemble(t,&regs[t]);break;
4102     case SHIFTIMM:
4103       shiftimm_assemble(t,&regs[t]);break;
4104     case LOAD:
4105       load_assemble(t,&regs[t]);break;
4106     case LOADLR:
4107       loadlr_assemble(t,&regs[t]);break;
4108     case STORE:
4109       store_assemble(t,&regs[t]);break;
4110     case STORELR:
4111       storelr_assemble(t,&regs[t]);break;
4112     case COP0:
4113       cop0_assemble(t,&regs[t]);break;
4114     case COP1:
4115       cop1_assemble(t,&regs[t]);break;
4116     case C1LS:
4117       c1ls_assemble(t,&regs[t]);break;
4118     case COP2:
4119       cop2_assemble(t,&regs[t]);break;
4120     case C2LS:
4121       c2ls_assemble(t,&regs[t]);break;
4122     case C2OP:
4123       c2op_assemble(t,&regs[t]);break;
4124     case MULTDIV:
4125       multdiv_assemble(t,&regs[t]);break;
4126     case MOV:
4127       mov_assemble(t,&regs[t]);break;
4128     case SYSCALL:
4129     case HLECALL:
4130     case INTCALL:
4131     case SPAN:
4132     case UJUMP:
4133     case RJUMP:
4134     case CJUMP:
4135     case SJUMP:
4136       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
4137   }
4138   store_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4139   load_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4140   if(internal_branch(ba[i]+4))
4141     assem_debug("branch: internal\n");
4142   else
4143     assem_debug("branch: external\n");
4144   assert(internal_branch(ba[i]+4));
4145   add_to_linker(out,ba[i]+4,internal_branch(ba[i]+4));
4146   emit_jmp(0);
4147 }
4148
4149 void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert)
4150 {
4151   int count;
4152   void *jaddr;
4153   void *idle=NULL;
4154   int t=0;
4155   if(itype[i]==RJUMP)
4156   {
4157     *adj=0;
4158   }
4159   //if(ba[i]>=start && ba[i]<(start+slen*4))
4160   if(internal_branch(ba[i]))
4161   {
4162     t=(ba[i]-start)>>2;
4163     if(is_ds[t]) *adj=-1; // Branch into delay slot adds an extra cycle
4164     else *adj=ccadj[t];
4165   }
4166   else
4167   {
4168     *adj=0;
4169   }
4170   count=ccadj[i];
4171   if(taken==TAKEN && i==(ba[i]-start)>>2 && source[i+1]==0) {
4172     // Idle loop
4173     if(count&1) emit_addimm_and_set_flags(2*(count+2),HOST_CCREG);
4174     idle=out;
4175     //emit_subfrommem(&idlecount,HOST_CCREG); // Count idle cycles
4176     emit_andimm(HOST_CCREG,3,HOST_CCREG);
4177     jaddr=out;
4178     emit_jmp(0);
4179   }
4180   else if(*adj==0||invert) {
4181     int cycles=CLOCK_ADJUST(count+2);
4182     // faster loop HACK
4183     if (t&&*adj) {
4184       int rel=t-i;
4185       if(-NO_CYCLE_PENALTY_THR<rel&&rel<0)
4186         cycles=CLOCK_ADJUST(*adj)+count+2-*adj;
4187     }
4188     emit_addimm_and_set_flags(cycles,HOST_CCREG);
4189     jaddr=out;
4190     emit_jns(0);
4191   }
4192   else
4193   {
4194     emit_cmpimm(HOST_CCREG,-CLOCK_ADJUST(count+2));
4195     jaddr=out;
4196     emit_jns(0);
4197   }
4198   add_stub(CC_STUB,jaddr,idle?idle:out,(*adj==0||invert||idle)?0:(count+2),i,addr,taken,0);
4199 }
4200
4201 static void do_ccstub(int n)
4202 {
4203   literal_pool(256);
4204   assem_debug("do_ccstub %lx\n",start+stubs[n].b*4);
4205   set_jump_target(stubs[n].addr, out);
4206   int i=stubs[n].b;
4207   if(stubs[n].d==NULLDS) {
4208     // Delay slot instruction is nullified ("likely" branch)
4209     wb_dirtys(regs[i].regmap,regs[i].dirty);
4210   }
4211   else if(stubs[n].d!=TAKEN) {
4212     wb_dirtys(branch_regs[i].regmap,branch_regs[i].dirty);
4213   }
4214   else {
4215     if(internal_branch(ba[i]))
4216       wb_needed_dirtys(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4217   }
4218   if(stubs[n].c!=-1)
4219   {
4220     // Save PC as return address
4221     emit_movimm(stubs[n].c,EAX);
4222     emit_writeword(EAX,&pcaddr);
4223   }
4224   else
4225   {
4226     // Return address depends on which way the branch goes
4227     if(itype[i]==CJUMP||itype[i]==SJUMP)
4228     {
4229       int s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4230       int s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4231       if(rs1[i]==0)
4232       {
4233         s1l=s2l;
4234         s2l=-1;
4235       }
4236       else if(rs2[i]==0)
4237       {
4238         s2l=-1;
4239       }
4240       assert(s1l>=0);
4241       #ifdef DESTRUCTIVE_WRITEBACK
4242       if(rs1[i]) {
4243         if((branch_regs[i].dirty>>s1l)&&1)
4244           emit_loadreg(rs1[i],s1l);
4245       }
4246       else {
4247         if((branch_regs[i].dirty>>s1l)&1)
4248           emit_loadreg(rs2[i],s1l);
4249       }
4250       if(s2l>=0)
4251         if((branch_regs[i].dirty>>s2l)&1)
4252           emit_loadreg(rs2[i],s2l);
4253       #endif
4254       int hr=0;
4255       int addr=-1,alt=-1,ntaddr=-1;
4256       while(hr<HOST_REGS)
4257       {
4258         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4259            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4260            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4261         {
4262           addr=hr++;break;
4263         }
4264         hr++;
4265       }
4266       while(hr<HOST_REGS)
4267       {
4268         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4269            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4270            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4271         {
4272           alt=hr++;break;
4273         }
4274         hr++;
4275       }
4276       if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
4277       {
4278         while(hr<HOST_REGS)
4279         {
4280           if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4281              (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4282              (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4283           {
4284             ntaddr=hr;break;
4285           }
4286           hr++;
4287         }
4288         assert(hr<HOST_REGS);
4289       }
4290       if((opcode[i]&0x2f)==4) // BEQ
4291       {
4292         #ifdef HAVE_CMOV_IMM
4293         if(s2l>=0) emit_cmp(s1l,s2l);
4294         else emit_test(s1l,s1l);
4295         emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
4296         #else
4297         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4298         if(s2l>=0) emit_cmp(s1l,s2l);
4299         else emit_test(s1l,s1l);
4300         emit_cmovne_reg(alt,addr);
4301         #endif
4302       }
4303       if((opcode[i]&0x2f)==5) // BNE
4304       {
4305         #ifdef HAVE_CMOV_IMM
4306         if(s2l>=0) emit_cmp(s1l,s2l);
4307         else emit_test(s1l,s1l);
4308         emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
4309         #else
4310         emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
4311         if(s2l>=0) emit_cmp(s1l,s2l);
4312         else emit_test(s1l,s1l);
4313         emit_cmovne_reg(alt,addr);
4314         #endif
4315       }
4316       if((opcode[i]&0x2f)==6) // BLEZ
4317       {
4318         //emit_movimm(ba[i],alt);
4319         //emit_movimm(start+i*4+8,addr);
4320         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4321         emit_cmpimm(s1l,1);
4322         emit_cmovl_reg(alt,addr);
4323       }
4324       if((opcode[i]&0x2f)==7) // BGTZ
4325       {
4326         //emit_movimm(ba[i],addr);
4327         //emit_movimm(start+i*4+8,ntaddr);
4328         emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
4329         emit_cmpimm(s1l,1);
4330         emit_cmovl_reg(ntaddr,addr);
4331       }
4332       if((opcode[i]==1)&&(opcode2[i]&0x2D)==0) // BLTZ
4333       {
4334         //emit_movimm(ba[i],alt);
4335         //emit_movimm(start+i*4+8,addr);
4336         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4337         emit_test(s1l,s1l);
4338         emit_cmovs_reg(alt,addr);
4339       }
4340       if((opcode[i]==1)&&(opcode2[i]&0x2D)==1) // BGEZ
4341       {
4342         //emit_movimm(ba[i],addr);
4343         //emit_movimm(start+i*4+8,alt);
4344         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4345         emit_test(s1l,s1l);
4346         emit_cmovs_reg(alt,addr);
4347       }
4348       if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
4349         if(source[i]&0x10000) // BC1T
4350         {
4351           //emit_movimm(ba[i],alt);
4352           //emit_movimm(start+i*4+8,addr);
4353           emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4354           emit_testimm(s1l,0x800000);
4355           emit_cmovne_reg(alt,addr);
4356         }
4357         else // BC1F
4358         {
4359           //emit_movimm(ba[i],addr);
4360           //emit_movimm(start+i*4+8,alt);
4361           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4362           emit_testimm(s1l,0x800000);
4363           emit_cmovne_reg(alt,addr);
4364         }
4365       }
4366       emit_writeword(addr,&pcaddr);
4367     }
4368     else
4369     if(itype[i]==RJUMP)
4370     {
4371       int r=get_reg(branch_regs[i].regmap,rs1[i]);
4372       if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4373         r=get_reg(branch_regs[i].regmap,RTEMP);
4374       }
4375       emit_writeword(r,&pcaddr);
4376     }
4377     else {SysPrintf("Unknown branch type in do_ccstub\n");exit(1);}
4378   }
4379   // Update cycle count
4380   assert(branch_regs[i].regmap[HOST_CCREG]==CCREG||branch_regs[i].regmap[HOST_CCREG]==-1);
4381   if(stubs[n].a) emit_addimm(HOST_CCREG,CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4382   emit_call(cc_interrupt);
4383   if(stubs[n].a) emit_addimm(HOST_CCREG,-CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4384   if(stubs[n].d==TAKEN) {
4385     if(internal_branch(ba[i]))
4386       load_needed_regs(branch_regs[i].regmap,regs[(ba[i]-start)>>2].regmap_entry);
4387     else if(itype[i]==RJUMP) {
4388       if(get_reg(branch_regs[i].regmap,RTEMP)>=0)
4389         emit_readword(&pcaddr,get_reg(branch_regs[i].regmap,RTEMP));
4390       else
4391         emit_loadreg(rs1[i],get_reg(branch_regs[i].regmap,rs1[i]));
4392     }
4393   }else if(stubs[n].d==NOTTAKEN) {
4394     if(i<slen-2) load_needed_regs(branch_regs[i].regmap,regmap_pre[i+2]);
4395     else load_all_regs(branch_regs[i].regmap);
4396   }else if(stubs[n].d==NULLDS) {
4397     // Delay slot instruction is nullified ("likely" branch)
4398     if(i<slen-2) load_needed_regs(regs[i].regmap,regmap_pre[i+2]);
4399     else load_all_regs(regs[i].regmap);
4400   }else{
4401     load_all_regs(branch_regs[i].regmap);
4402   }
4403   emit_jmp(stubs[n].retaddr);
4404 }
4405
4406 static void add_to_linker(void *addr, u_int target, int ext)
4407 {
4408   assert(linkcount < ARRAY_SIZE(link_addr));
4409   link_addr[linkcount].addr = addr;
4410   link_addr[linkcount].target = target;
4411   link_addr[linkcount].ext = ext;
4412   linkcount++;
4413 }
4414
4415 static void ujump_assemble_write_ra(int i)
4416 {
4417   int rt;
4418   unsigned int return_address;
4419   rt=get_reg(branch_regs[i].regmap,31);
4420   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4421   //assert(rt>=0);
4422   return_address=start+i*4+8;
4423   if(rt>=0) {
4424     #ifdef USE_MINI_HT
4425     if(internal_branch(return_address)&&rt1[i+1]!=31) {
4426       int temp=-1; // note: must be ds-safe
4427       #ifdef HOST_TEMPREG
4428       temp=HOST_TEMPREG;
4429       #endif
4430       if(temp>=0) do_miniht_insert(return_address,rt,temp);
4431       else emit_movimm(return_address,rt);
4432     }
4433     else
4434     #endif
4435     {
4436       #ifdef REG_PREFETCH
4437       if(temp>=0)
4438       {
4439         if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4440       }
4441       #endif
4442       emit_movimm(return_address,rt); // PC into link register
4443       #ifdef IMM_PREFETCH
4444       emit_prefetch(hash_table_get(return_address));
4445       #endif
4446     }
4447   }
4448 }
4449
4450 void ujump_assemble(int i,struct regstat *i_regs)
4451 {
4452   int ra_done=0;
4453   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4454   address_generation(i+1,i_regs,regs[i].regmap_entry);
4455   #ifdef REG_PREFETCH
4456   int temp=get_reg(branch_regs[i].regmap,PTEMP);
4457   if(rt1[i]==31&&temp>=0)
4458   {
4459     signed char *i_regmap=i_regs->regmap;
4460     int return_address=start+i*4+8;
4461     if(get_reg(branch_regs[i].regmap,31)>0)
4462     if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4463   }
4464   #endif
4465   if(rt1[i]==31&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4466     ujump_assemble_write_ra(i); // writeback ra for DS
4467     ra_done=1;
4468   }
4469   ds_assemble(i+1,i_regs);
4470   uint64_t bc_unneeded=branch_regs[i].u;
4471   bc_unneeded|=1|(1LL<<rt1[i]);
4472   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4473   load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4474   if(!ra_done&&rt1[i]==31)
4475     ujump_assemble_write_ra(i);
4476   int cc,adj;
4477   cc=get_reg(branch_regs[i].regmap,CCREG);
4478   assert(cc==HOST_CCREG);
4479   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4480   #ifdef REG_PREFETCH
4481   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4482   #endif
4483   do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4484   if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4485   load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4486   if(internal_branch(ba[i]))
4487     assem_debug("branch: internal\n");
4488   else
4489     assem_debug("branch: external\n");
4490   if(internal_branch(ba[i])&&is_ds[(ba[i]-start)>>2]) {
4491     ds_assemble_entry(i);
4492   }
4493   else {
4494     add_to_linker(out,ba[i],internal_branch(ba[i]));
4495     emit_jmp(0);
4496   }
4497 }
4498
4499 static void rjump_assemble_write_ra(int i)
4500 {
4501   int rt,return_address;
4502   assert(rt1[i+1]!=rt1[i]);
4503   assert(rt2[i+1]!=rt1[i]);
4504   rt=get_reg(branch_regs[i].regmap,rt1[i]);
4505   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4506   assert(rt>=0);
4507   return_address=start+i*4+8;
4508   #ifdef REG_PREFETCH
4509   if(temp>=0)
4510   {
4511     if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4512   }
4513   #endif
4514   emit_movimm(return_address,rt); // PC into link register
4515   #ifdef IMM_PREFETCH
4516   emit_prefetch(hash_table_get(return_address));
4517   #endif
4518 }
4519
4520 void rjump_assemble(int i,struct regstat *i_regs)
4521 {
4522   int temp;
4523   int rs,cc;
4524   int ra_done=0;
4525   rs=get_reg(branch_regs[i].regmap,rs1[i]);
4526   assert(rs>=0);
4527   if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4528     // Delay slot abuse, make a copy of the branch address register
4529     temp=get_reg(branch_regs[i].regmap,RTEMP);
4530     assert(temp>=0);
4531     assert(regs[i].regmap[temp]==RTEMP);
4532     emit_mov(rs,temp);
4533     rs=temp;
4534   }
4535   address_generation(i+1,i_regs,regs[i].regmap_entry);
4536   #ifdef REG_PREFETCH
4537   if(rt1[i]==31)
4538   {
4539     if((temp=get_reg(branch_regs[i].regmap,PTEMP))>=0) {
4540       signed char *i_regmap=i_regs->regmap;
4541       int return_address=start+i*4+8;
4542       if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4543     }
4544   }
4545   #endif
4546   #ifdef USE_MINI_HT
4547   if(rs1[i]==31) {
4548     int rh=get_reg(regs[i].regmap,RHASH);
4549     if(rh>=0) do_preload_rhash(rh);
4550   }
4551   #endif
4552   if(rt1[i]!=0&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4553     rjump_assemble_write_ra(i);
4554     ra_done=1;
4555   }
4556   ds_assemble(i+1,i_regs);
4557   uint64_t bc_unneeded=branch_regs[i].u;
4558   bc_unneeded|=1|(1LL<<rt1[i]);
4559   bc_unneeded&=~(1LL<<rs1[i]);
4560   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4561   load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],CCREG);
4562   if(!ra_done&&rt1[i]!=0)
4563     rjump_assemble_write_ra(i);
4564   cc=get_reg(branch_regs[i].regmap,CCREG);
4565   assert(cc==HOST_CCREG);
4566   (void)cc;
4567   #ifdef USE_MINI_HT
4568   int rh=get_reg(branch_regs[i].regmap,RHASH);
4569   int ht=get_reg(branch_regs[i].regmap,RHTBL);
4570   if(rs1[i]==31) {
4571     if(regs[i].regmap[rh]!=RHASH) do_preload_rhash(rh);
4572     do_preload_rhtbl(ht);
4573     do_rhash(rs,rh);
4574   }
4575   #endif
4576   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4577   #ifdef DESTRUCTIVE_WRITEBACK
4578   if((branch_regs[i].dirty>>rs)&1) {
4579     if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
4580       emit_loadreg(rs1[i],rs);
4581     }
4582   }
4583   #endif
4584   #ifdef REG_PREFETCH
4585   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4586   #endif
4587   #ifdef USE_MINI_HT
4588   if(rs1[i]==31) {
4589     do_miniht_load(ht,rh);
4590   }
4591   #endif
4592   //do_cc(i,branch_regs[i].regmap,&adj,-1,TAKEN);
4593   //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen
4594   //assert(adj==0);
4595   emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4596   add_stub(CC_STUB,out,jump_vaddr_reg[rs],0,i,-1,TAKEN,0);
4597   if(itype[i+1]==COP0&&(source[i+1]&0x3f)==0x10)
4598     // special case for RFE
4599     emit_jmp(0);
4600   else
4601     emit_jns(0);
4602   //load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4603   #ifdef USE_MINI_HT
4604   if(rs1[i]==31) {
4605     do_miniht_jump(rs,rh,ht);
4606   }
4607   else
4608   #endif
4609   {
4610     emit_jmp(jump_vaddr_reg[rs]);
4611   }
4612   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4613   if(rt1[i]!=31&&i<slen-2&&(((u_int)out)&7)) emit_mov(13,13);
4614   #endif
4615 }
4616
4617 void cjump_assemble(int i,struct regstat *i_regs)
4618 {
4619   signed char *i_regmap=i_regs->regmap;
4620   int cc;
4621   int match;
4622   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4623   assem_debug("match=%d\n",match);
4624   int s1l,s2l;
4625   int unconditional=0,nop=0;
4626   int invert=0;
4627   int internal=internal_branch(ba[i]);
4628   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4629   if(!match) invert=1;
4630   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4631   if(i>(ba[i]-start)>>2) invert=1;
4632   #endif
4633
4634   if(ooo[i]) {
4635     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4636     s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4637   }
4638   else {
4639     s1l=get_reg(i_regmap,rs1[i]);
4640     s2l=get_reg(i_regmap,rs2[i]);
4641   }
4642   if(rs1[i]==0&&rs2[i]==0)
4643   {
4644     if(opcode[i]&1) nop=1;
4645     else unconditional=1;
4646     //assert(opcode[i]!=5);
4647     //assert(opcode[i]!=7);
4648     //assert(opcode[i]!=0x15);
4649     //assert(opcode[i]!=0x17);
4650   }
4651   else if(rs1[i]==0)
4652   {
4653     s1l=s2l;
4654     s2l=-1;
4655   }
4656   else if(rs2[i]==0)
4657   {
4658     s2l=-1;
4659   }
4660
4661   if(ooo[i]) {
4662     // Out of order execution (delay slot first)
4663     //printf("OOOE\n");
4664     address_generation(i+1,i_regs,regs[i].regmap_entry);
4665     ds_assemble(i+1,i_regs);
4666     int adj;
4667     uint64_t bc_unneeded=branch_regs[i].u;
4668     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
4669     bc_unneeded|=1;
4670     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4671     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs2[i]);
4672     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4673     cc=get_reg(branch_regs[i].regmap,CCREG);
4674     assert(cc==HOST_CCREG);
4675     if(unconditional)
4676       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4677     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
4678     //assem_debug("cycle count (adj)\n");
4679     if(unconditional) {
4680       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4681       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
4682         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4683         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4684         if(internal)
4685           assem_debug("branch: internal\n");
4686         else
4687           assem_debug("branch: external\n");
4688         if(internal&&is_ds[(ba[i]-start)>>2]) {
4689           ds_assemble_entry(i);
4690         }
4691         else {
4692           add_to_linker(out,ba[i],internal);
4693           emit_jmp(0);
4694         }
4695         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4696         if(((u_int)out)&7) emit_addnop(0);
4697         #endif
4698       }
4699     }
4700     else if(nop) {
4701       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4702       void *jaddr=out;
4703       emit_jns(0);
4704       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4705     }
4706     else {
4707       void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
4708       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
4709       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4710
4711       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4712       assert(s1l>=0);
4713       if(opcode[i]==4) // BEQ
4714       {
4715         if(s2l>=0) emit_cmp(s1l,s2l);
4716         else emit_test(s1l,s1l);
4717         if(invert){
4718           nottaken=out;
4719           emit_jne((void *)1l);
4720         }else{
4721           add_to_linker(out,ba[i],internal);
4722           emit_jeq(0);
4723         }
4724       }
4725       if(opcode[i]==5) // BNE
4726       {
4727         if(s2l>=0) emit_cmp(s1l,s2l);
4728         else emit_test(s1l,s1l);
4729         if(invert){
4730           nottaken=out;
4731           emit_jeq(1);
4732         }else{
4733           add_to_linker(out,ba[i],internal);
4734           emit_jne(0);
4735         }
4736       }
4737       if(opcode[i]==6) // BLEZ
4738       {
4739         emit_cmpimm(s1l,1);
4740         if(invert){
4741           nottaken=out;
4742           emit_jge(1);
4743         }else{
4744           add_to_linker(out,ba[i],internal);
4745           emit_jl(0);
4746         }
4747       }
4748       if(opcode[i]==7) // BGTZ
4749       {
4750         emit_cmpimm(s1l,1);
4751         if(invert){
4752           nottaken=out;
4753           emit_jl(1);
4754         }else{
4755           add_to_linker(out,ba[i],internal);
4756           emit_jge(0);
4757         }
4758       }
4759       if(invert) {
4760         if(taken) set_jump_target(taken, out);
4761         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4762         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
4763           if(adj) {
4764             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
4765             add_to_linker(out,ba[i],internal);
4766           }else{
4767             emit_addnop(13);
4768             add_to_linker(out,ba[i],internal*2);
4769           }
4770           emit_jmp(0);
4771         }else
4772         #endif
4773         {
4774           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
4775           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4776           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4777           if(internal)
4778             assem_debug("branch: internal\n");
4779           else
4780             assem_debug("branch: external\n");
4781           if(internal&&is_ds[(ba[i]-start)>>2]) {
4782             ds_assemble_entry(i);
4783           }
4784           else {
4785             add_to_linker(out,ba[i],internal);
4786             emit_jmp(0);
4787           }
4788         }
4789         set_jump_target(nottaken, out);
4790       }
4791
4792       if(nottaken1) set_jump_target(nottaken1, out);
4793       if(adj) {
4794         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
4795       }
4796     } // (!unconditional)
4797   } // if(ooo)
4798   else
4799   {
4800     // In-order execution (branch first)
4801     //if(likely[i]) printf("IOL\n");
4802     //else
4803     //printf("IOE\n");
4804     void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
4805     if(!unconditional&&!nop) {
4806       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4807       assert(s1l>=0);
4808       if((opcode[i]&0x2f)==4) // BEQ
4809       {
4810         if(s2l>=0) emit_cmp(s1l,s2l);
4811         else emit_test(s1l,s1l);
4812         nottaken=out;
4813         emit_jne((void *)2l);
4814       }
4815       if((opcode[i]&0x2f)==5) // BNE
4816       {
4817         if(s2l>=0) emit_cmp(s1l,s2l);
4818         else emit_test(s1l,s1l);
4819         nottaken=out;
4820         emit_jeq(2);
4821       }
4822       if((opcode[i]&0x2f)==6) // BLEZ
4823       {
4824         emit_cmpimm(s1l,1);
4825         nottaken=out;
4826         emit_jge(2);
4827       }
4828       if((opcode[i]&0x2f)==7) // BGTZ
4829       {
4830         emit_cmpimm(s1l,1);
4831         nottaken=out;
4832         emit_jl(2);
4833       }
4834     } // if(!unconditional)
4835     int adj;
4836     uint64_t ds_unneeded=branch_regs[i].u;
4837     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
4838     ds_unneeded|=1;
4839     // branch taken
4840     if(!nop) {
4841       if(taken) set_jump_target(taken, out);
4842       assem_debug("1:\n");
4843       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
4844       // load regs
4845       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
4846       address_generation(i+1,&branch_regs[i],0);
4847       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
4848       ds_assemble(i+1,&branch_regs[i]);
4849       cc=get_reg(branch_regs[i].regmap,CCREG);
4850       if(cc==-1) {
4851         emit_loadreg(CCREG,cc=HOST_CCREG);
4852         // CHECK: Is the following instruction (fall thru) allocated ok?
4853       }
4854       assert(cc==HOST_CCREG);
4855       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4856       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
4857       assem_debug("cycle count (adj)\n");
4858       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4859       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4860       if(internal)
4861         assem_debug("branch: internal\n");
4862       else
4863         assem_debug("branch: external\n");
4864       if(internal&&is_ds[(ba[i]-start)>>2]) {
4865         ds_assemble_entry(i);
4866       }
4867       else {
4868         add_to_linker(out,ba[i],internal);
4869         emit_jmp(0);
4870       }
4871     }
4872     // branch not taken
4873     if(!unconditional) {
4874       if(nottaken1) set_jump_target(nottaken1, out);
4875       set_jump_target(nottaken, out);
4876       assem_debug("2:\n");
4877       if(!likely[i]) {
4878         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
4879         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
4880         address_generation(i+1,&branch_regs[i],0);
4881         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4882         ds_assemble(i+1,&branch_regs[i]);
4883       }
4884       cc=get_reg(branch_regs[i].regmap,CCREG);
4885       if(cc==-1&&!likely[i]) {
4886         // Cycle count isn't in a register, temporarily load it then write it out
4887         emit_loadreg(CCREG,HOST_CCREG);
4888         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4889         void *jaddr=out;
4890         emit_jns(0);
4891         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4892         emit_storereg(CCREG,HOST_CCREG);
4893       }
4894       else{
4895         cc=get_reg(i_regmap,CCREG);
4896         assert(cc==HOST_CCREG);
4897         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4898         void *jaddr=out;
4899         emit_jns(0);
4900         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
4901       }
4902     }
4903   }
4904 }
4905
4906 void sjump_assemble(int i,struct regstat *i_regs)
4907 {
4908   signed char *i_regmap=i_regs->regmap;
4909   int cc;
4910   int match;
4911   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4912   assem_debug("smatch=%d\n",match);
4913   int s1l;
4914   int unconditional=0,nevertaken=0;
4915   int invert=0;
4916   int internal=internal_branch(ba[i]);
4917   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4918   if(!match) invert=1;
4919   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4920   if(i>(ba[i]-start)>>2) invert=1;
4921   #endif
4922
4923   //if(opcode2[i]>=0x10) return; // FIXME (BxxZAL)
4924   //assert(opcode2[i]<0x10||rs1[i]==0); // FIXME (BxxZAL)
4925
4926   if(ooo[i]) {
4927     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4928   }
4929   else {
4930     s1l=get_reg(i_regmap,rs1[i]);
4931   }
4932   if(rs1[i]==0)
4933   {
4934     if(opcode2[i]&1) unconditional=1;
4935     else nevertaken=1;
4936     // These are never taken (r0 is never less than zero)
4937     //assert(opcode2[i]!=0);
4938     //assert(opcode2[i]!=2);
4939     //assert(opcode2[i]!=0x10);
4940     //assert(opcode2[i]!=0x12);
4941   }
4942
4943   if(ooo[i]) {
4944     // Out of order execution (delay slot first)
4945     //printf("OOOE\n");
4946     address_generation(i+1,i_regs,regs[i].regmap_entry);
4947     ds_assemble(i+1,i_regs);
4948     int adj;
4949     uint64_t bc_unneeded=branch_regs[i].u;
4950     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
4951     bc_unneeded|=1;
4952     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4953     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs1[i]);
4954     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4955     if(rt1[i]==31) {
4956       int rt,return_address;
4957       rt=get_reg(branch_regs[i].regmap,31);
4958       assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4959       if(rt>=0) {
4960         // Save the PC even if the branch is not taken
4961         return_address=start+i*4+8;
4962         emit_movimm(return_address,rt); // PC into link register
4963         #ifdef IMM_PREFETCH
4964         if(!nevertaken) emit_prefetch(hash_table_get(return_address));
4965         #endif
4966       }
4967     }
4968     cc=get_reg(branch_regs[i].regmap,CCREG);
4969     assert(cc==HOST_CCREG);
4970     if(unconditional)
4971       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4972     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
4973     assem_debug("cycle count (adj)\n");
4974     if(unconditional) {
4975       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4976       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
4977         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4978         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4979         if(internal)
4980           assem_debug("branch: internal\n");
4981         else
4982           assem_debug("branch: external\n");
4983         if(internal&&is_ds[(ba[i]-start)>>2]) {
4984           ds_assemble_entry(i);
4985         }
4986         else {
4987           add_to_linker(out,ba[i],internal);
4988           emit_jmp(0);
4989         }
4990         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4991         if(((u_int)out)&7) emit_addnop(0);
4992         #endif
4993       }
4994     }
4995     else if(nevertaken) {
4996       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4997       void *jaddr=out;
4998       emit_jns(0);
4999       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5000     }
5001     else {
5002       void *nottaken = NULL;
5003       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5004       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5005       {
5006         assert(s1l>=0);
5007         if((opcode2[i]&0xf)==0) // BLTZ/BLTZAL
5008         {
5009           emit_test(s1l,s1l);
5010           if(invert){
5011             nottaken=out;
5012             emit_jns(1);
5013           }else{
5014             add_to_linker(out,ba[i],internal);
5015             emit_js(0);
5016           }
5017         }
5018         if((opcode2[i]&0xf)==1) // BGEZ/BLTZAL
5019         {
5020           emit_test(s1l,s1l);
5021           if(invert){
5022             nottaken=out;
5023             emit_js(1);
5024           }else{
5025             add_to_linker(out,ba[i],internal);
5026             emit_jns(0);
5027           }
5028         }
5029       }
5030
5031       if(invert) {
5032         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5033         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5034           if(adj) {
5035             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5036             add_to_linker(out,ba[i],internal);
5037           }else{
5038             emit_addnop(13);
5039             add_to_linker(out,ba[i],internal*2);
5040           }
5041           emit_jmp(0);
5042         }else
5043         #endif
5044         {
5045           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5046           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5047           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5048           if(internal)
5049             assem_debug("branch: internal\n");
5050           else
5051             assem_debug("branch: external\n");
5052           if(internal&&is_ds[(ba[i]-start)>>2]) {
5053             ds_assemble_entry(i);
5054           }
5055           else {
5056             add_to_linker(out,ba[i],internal);
5057             emit_jmp(0);
5058           }
5059         }
5060         set_jump_target(nottaken, out);
5061       }
5062
5063       if(adj) {
5064         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5065       }
5066     } // (!unconditional)
5067   } // if(ooo)
5068   else
5069   {
5070     // In-order execution (branch first)
5071     //printf("IOE\n");
5072     void *nottaken = NULL;
5073     if(rt1[i]==31) {
5074       int rt,return_address;
5075       rt=get_reg(branch_regs[i].regmap,31);
5076       if(rt>=0) {
5077         // Save the PC even if the branch is not taken
5078         return_address=start+i*4+8;
5079         emit_movimm(return_address,rt); // PC into link register
5080         #ifdef IMM_PREFETCH
5081         emit_prefetch(hash_table_get(return_address));
5082         #endif
5083       }
5084     }
5085     if(!unconditional) {
5086       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5087         assert(s1l>=0);
5088         if((opcode2[i]&0x0d)==0) // BLTZ/BLTZL/BLTZAL/BLTZALL
5089         {
5090           emit_test(s1l,s1l);
5091           nottaken=out;
5092           emit_jns(1);
5093         }
5094         if((opcode2[i]&0x0d)==1) // BGEZ/BGEZL/BGEZAL/BGEZALL
5095         {
5096           emit_test(s1l,s1l);
5097           nottaken=out;
5098           emit_js(1);
5099         }
5100     } // if(!unconditional)
5101     int adj;
5102     uint64_t ds_unneeded=branch_regs[i].u;
5103     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5104     ds_unneeded|=1;
5105     // branch taken
5106     if(!nevertaken) {
5107       //assem_debug("1:\n");
5108       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5109       // load regs
5110       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5111       address_generation(i+1,&branch_regs[i],0);
5112       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5113       ds_assemble(i+1,&branch_regs[i]);
5114       cc=get_reg(branch_regs[i].regmap,CCREG);
5115       if(cc==-1) {
5116         emit_loadreg(CCREG,cc=HOST_CCREG);
5117         // CHECK: Is the following instruction (fall thru) allocated ok?
5118       }
5119       assert(cc==HOST_CCREG);
5120       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5121       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5122       assem_debug("cycle count (adj)\n");
5123       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5124       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5125       if(internal)
5126         assem_debug("branch: internal\n");
5127       else
5128         assem_debug("branch: external\n");
5129       if(internal&&is_ds[(ba[i]-start)>>2]) {
5130         ds_assemble_entry(i);
5131       }
5132       else {
5133         add_to_linker(out,ba[i],internal);
5134         emit_jmp(0);
5135       }
5136     }
5137     // branch not taken
5138     if(!unconditional) {
5139       set_jump_target(nottaken, out);
5140       assem_debug("1:\n");
5141       if(!likely[i]) {
5142         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5143         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5144         address_generation(i+1,&branch_regs[i],0);
5145         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5146         ds_assemble(i+1,&branch_regs[i]);
5147       }
5148       cc=get_reg(branch_regs[i].regmap,CCREG);
5149       if(cc==-1&&!likely[i]) {
5150         // Cycle count isn't in a register, temporarily load it then write it out
5151         emit_loadreg(CCREG,HOST_CCREG);
5152         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5153         void *jaddr=out;
5154         emit_jns(0);
5155         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5156         emit_storereg(CCREG,HOST_CCREG);
5157       }
5158       else{
5159         cc=get_reg(i_regmap,CCREG);
5160         assert(cc==HOST_CCREG);
5161         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5162         void *jaddr=out;
5163         emit_jns(0);
5164         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5165       }
5166     }
5167   }
5168 }
5169
5170 static void pagespan_assemble(int i,struct regstat *i_regs)
5171 {
5172   int s1l=get_reg(i_regs->regmap,rs1[i]);
5173   int s2l=get_reg(i_regs->regmap,rs2[i]);
5174   void *taken = NULL;
5175   void *nottaken = NULL;
5176   int unconditional=0;
5177   if(rs1[i]==0)
5178   {
5179     s1l=s2l;
5180     s2l=-1;
5181   }
5182   else if(rs2[i]==0)
5183   {
5184     s2l=-1;
5185   }
5186   int hr=0;
5187   int addr=-1,alt=-1,ntaddr=-1;
5188   if(i_regs->regmap[HOST_BTREG]<0) {addr=HOST_BTREG;}
5189   else {
5190     while(hr<HOST_REGS)
5191     {
5192       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
5193          (i_regs->regmap[hr]&63)!=rs1[i] &&
5194          (i_regs->regmap[hr]&63)!=rs2[i] )
5195       {
5196         addr=hr++;break;
5197       }
5198       hr++;
5199     }
5200   }
5201   while(hr<HOST_REGS)
5202   {
5203     if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5204        (i_regs->regmap[hr]&63)!=rs1[i] &&
5205        (i_regs->regmap[hr]&63)!=rs2[i] )
5206     {
5207       alt=hr++;break;
5208     }
5209     hr++;
5210   }
5211   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
5212   {
5213     while(hr<HOST_REGS)
5214     {
5215       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5216          (i_regs->regmap[hr]&63)!=rs1[i] &&
5217          (i_regs->regmap[hr]&63)!=rs2[i] )
5218       {
5219         ntaddr=hr;break;
5220       }
5221       hr++;
5222     }
5223   }
5224   assert(hr<HOST_REGS);
5225   if((opcode[i]&0x2e)==4||opcode[i]==0x11) { // BEQ/BNE/BEQL/BNEL/BC1
5226     load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
5227   }
5228   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5229   if(opcode[i]==2) // J
5230   {
5231     unconditional=1;
5232   }
5233   if(opcode[i]==3) // JAL
5234   {
5235     // TODO: mini_ht
5236     int rt=get_reg(i_regs->regmap,31);
5237     emit_movimm(start+i*4+8,rt);
5238     unconditional=1;
5239   }
5240   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
5241   {
5242     emit_mov(s1l,addr);
5243     if(opcode2[i]==9) // JALR
5244     {
5245       int rt=get_reg(i_regs->regmap,rt1[i]);
5246       emit_movimm(start+i*4+8,rt);
5247     }
5248   }
5249   if((opcode[i]&0x3f)==4) // BEQ
5250   {
5251     if(rs1[i]==rs2[i])
5252     {
5253       unconditional=1;
5254     }
5255     else
5256     #ifdef HAVE_CMOV_IMM
5257     if(1) {
5258       if(s2l>=0) emit_cmp(s1l,s2l);
5259       else emit_test(s1l,s1l);
5260       emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
5261     }
5262     else
5263     #endif
5264     {
5265       assert(s1l>=0);
5266       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5267       if(s2l>=0) emit_cmp(s1l,s2l);
5268       else emit_test(s1l,s1l);
5269       emit_cmovne_reg(alt,addr);
5270     }
5271   }
5272   if((opcode[i]&0x3f)==5) // BNE
5273   {
5274     #ifdef HAVE_CMOV_IMM
5275     if(s2l>=0) emit_cmp(s1l,s2l);
5276     else emit_test(s1l,s1l);
5277     emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
5278     #else
5279     assert(s1l>=0);
5280     emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
5281     if(s2l>=0) emit_cmp(s1l,s2l);
5282     else emit_test(s1l,s1l);
5283     emit_cmovne_reg(alt,addr);
5284     #endif
5285   }
5286   if((opcode[i]&0x3f)==0x14) // BEQL
5287   {
5288     if(s2l>=0) emit_cmp(s1l,s2l);
5289     else emit_test(s1l,s1l);
5290     if(nottaken) set_jump_target(nottaken, out);
5291     nottaken=out;
5292     emit_jne(0);
5293   }
5294   if((opcode[i]&0x3f)==0x15) // BNEL
5295   {
5296     if(s2l>=0) emit_cmp(s1l,s2l);
5297     else emit_test(s1l,s1l);
5298     nottaken=out;
5299     emit_jeq(0);
5300     if(taken) set_jump_target(taken, out);
5301   }
5302   if((opcode[i]&0x3f)==6) // BLEZ
5303   {
5304     emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5305     emit_cmpimm(s1l,1);
5306     emit_cmovl_reg(alt,addr);
5307   }
5308   if((opcode[i]&0x3f)==7) // BGTZ
5309   {
5310     emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
5311     emit_cmpimm(s1l,1);
5312     emit_cmovl_reg(ntaddr,addr);
5313   }
5314   if((opcode[i]&0x3f)==0x16) // BLEZL
5315   {
5316     assert((opcode[i]&0x3f)!=0x16);
5317   }
5318   if((opcode[i]&0x3f)==0x17) // BGTZL
5319   {
5320     assert((opcode[i]&0x3f)!=0x17);
5321   }
5322   assert(opcode[i]!=1); // BLTZ/BGEZ
5323
5324   //FIXME: Check CSREG
5325   if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
5326     if((source[i]&0x30000)==0) // BC1F
5327     {
5328       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5329       emit_testimm(s1l,0x800000);
5330       emit_cmovne_reg(alt,addr);
5331     }
5332     if((source[i]&0x30000)==0x10000) // BC1T
5333     {
5334       emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5335       emit_testimm(s1l,0x800000);
5336       emit_cmovne_reg(alt,addr);
5337     }
5338     if((source[i]&0x30000)==0x20000) // BC1FL
5339     {
5340       emit_testimm(s1l,0x800000);
5341       nottaken=out;
5342       emit_jne(0);
5343     }
5344     if((source[i]&0x30000)==0x30000) // BC1TL
5345     {
5346       emit_testimm(s1l,0x800000);
5347       nottaken=out;
5348       emit_jeq(0);
5349     }
5350   }
5351
5352   assert(i_regs->regmap[HOST_CCREG]==CCREG);
5353   wb_dirtys(regs[i].regmap,regs[i].dirty);
5354   if(likely[i]||unconditional)
5355   {
5356     emit_movimm(ba[i],HOST_BTREG);
5357   }
5358   else if(addr!=HOST_BTREG)
5359   {
5360     emit_mov(addr,HOST_BTREG);
5361   }
5362   void *branch_addr=out;
5363   emit_jmp(0);
5364   int target_addr=start+i*4+5;
5365   void *stub=out;
5366   void *compiled_target_addr=check_addr(target_addr);
5367   emit_extjump_ds(branch_addr, target_addr);
5368   if(compiled_target_addr) {
5369     set_jump_target(branch_addr, compiled_target_addr);
5370     add_link(target_addr,stub);
5371   }
5372   else set_jump_target(branch_addr, stub);
5373   if(likely[i]) {
5374     // Not-taken path
5375     set_jump_target(nottaken, out);
5376     wb_dirtys(regs[i].regmap,regs[i].dirty);
5377     void *branch_addr=out;
5378     emit_jmp(0);
5379     int target_addr=start+i*4+8;
5380     void *stub=out;
5381     void *compiled_target_addr=check_addr(target_addr);
5382     emit_extjump_ds(branch_addr, target_addr);
5383     if(compiled_target_addr) {
5384       set_jump_target(branch_addr, compiled_target_addr);
5385       add_link(target_addr,stub);
5386     }
5387     else set_jump_target(branch_addr, stub);
5388   }
5389 }
5390
5391 // Assemble the delay slot for the above
5392 static void pagespan_ds()
5393 {
5394   assem_debug("initial delay slot:\n");
5395   u_int vaddr=start+1;
5396   u_int page=get_page(vaddr);
5397   u_int vpage=get_vpage(vaddr);
5398   ll_add(jump_dirty+vpage,vaddr,(void *)out);
5399   do_dirty_stub_ds();
5400   ll_add(jump_in+page,vaddr,(void *)out);
5401   assert(regs[0].regmap_entry[HOST_CCREG]==CCREG);
5402   if(regs[0].regmap[HOST_CCREG]!=CCREG)
5403     wb_register(CCREG,regs[0].regmap_entry,regs[0].wasdirty);
5404   if(regs[0].regmap[HOST_BTREG]!=BTREG)
5405     emit_writeword(HOST_BTREG,&branch_target);
5406   load_regs(regs[0].regmap_entry,regs[0].regmap,rs1[0],rs2[0]);
5407   address_generation(0,&regs[0],regs[0].regmap_entry);
5408   if(itype[0]==STORE||itype[0]==STORELR||(opcode[0]&0x3b)==0x39||(opcode[0]&0x3b)==0x3a)
5409     load_regs(regs[0].regmap_entry,regs[0].regmap,INVCP,INVCP);
5410   is_delayslot=0;
5411   switch(itype[0]) {
5412     case ALU:
5413       alu_assemble(0,&regs[0]);break;
5414     case IMM16:
5415       imm16_assemble(0,&regs[0]);break;
5416     case SHIFT:
5417       shift_assemble(0,&regs[0]);break;
5418     case SHIFTIMM:
5419       shiftimm_assemble(0,&regs[0]);break;
5420     case LOAD:
5421       load_assemble(0,&regs[0]);break;
5422     case LOADLR:
5423       loadlr_assemble(0,&regs[0]);break;
5424     case STORE:
5425       store_assemble(0,&regs[0]);break;
5426     case STORELR:
5427       storelr_assemble(0,&regs[0]);break;
5428     case COP0:
5429       cop0_assemble(0,&regs[0]);break;
5430     case COP1:
5431       cop1_assemble(0,&regs[0]);break;
5432     case C1LS:
5433       c1ls_assemble(0,&regs[0]);break;
5434     case COP2:
5435       cop2_assemble(0,&regs[0]);break;
5436     case C2LS:
5437       c2ls_assemble(0,&regs[0]);break;
5438     case C2OP:
5439       c2op_assemble(0,&regs[0]);break;
5440     case MULTDIV:
5441       multdiv_assemble(0,&regs[0]);break;
5442     case MOV:
5443       mov_assemble(0,&regs[0]);break;
5444     case SYSCALL:
5445     case HLECALL:
5446     case INTCALL:
5447     case SPAN:
5448     case UJUMP:
5449     case RJUMP:
5450     case CJUMP:
5451     case SJUMP:
5452       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
5453   }
5454   int btaddr=get_reg(regs[0].regmap,BTREG);
5455   if(btaddr<0) {
5456     btaddr=get_reg(regs[0].regmap,-1);
5457     emit_readword(&branch_target,btaddr);
5458   }
5459   assert(btaddr!=HOST_CCREG);
5460   if(regs[0].regmap[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
5461 #ifdef HOST_IMM8
5462   emit_movimm(start+4,HOST_TEMPREG);
5463   emit_cmp(btaddr,HOST_TEMPREG);
5464 #else
5465   emit_cmpimm(btaddr,start+4);
5466 #endif
5467   void *branch = out;
5468   emit_jeq(0);
5469   store_regs_bt(regs[0].regmap,regs[0].dirty,-1);
5470   emit_jmp(jump_vaddr_reg[btaddr]);
5471   set_jump_target(branch, out);
5472   store_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5473   load_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5474 }
5475
5476 // Basic liveness analysis for MIPS registers
5477 void unneeded_registers(int istart,int iend,int r)
5478 {
5479   int i;
5480   uint64_t u,gte_u,b,gte_b;
5481   uint64_t temp_u,temp_gte_u=0;
5482   uint64_t gte_u_unknown=0;
5483   if(new_dynarec_hacks&NDHACK_GTE_UNNEEDED)
5484     gte_u_unknown=~0ll;
5485   if(iend==slen-1) {
5486     u=1;
5487     gte_u=gte_u_unknown;
5488   }else{
5489     //u=unneeded_reg[iend+1];
5490     u=1;
5491     gte_u=gte_unneeded[iend+1];
5492   }
5493
5494   for (i=iend;i>=istart;i--)
5495   {
5496     //printf("unneeded registers i=%d (%d,%d) r=%d\n",i,istart,iend,r);
5497     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5498     {
5499       // If subroutine call, flag return address as a possible branch target
5500       if(rt1[i]==31 && i<slen-2) bt[i+2]=1;
5501
5502       if(ba[i]<start || ba[i]>=(start+slen*4))
5503       {
5504         // Branch out of this block, flush all regs
5505         u=1;
5506         gte_u=gte_u_unknown;
5507         branch_unneeded_reg[i]=u;
5508         // Merge in delay slot
5509         u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5510         u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5511         u|=1;
5512         gte_u|=gte_rt[i+1];
5513         gte_u&=~gte_rs[i+1];
5514         // If branch is "likely" (and conditional)
5515         // then we skip the delay slot on the fall-thru path
5516         if(likely[i]) {
5517           if(i<slen-1) {
5518             u&=unneeded_reg[i+2];
5519             gte_u&=gte_unneeded[i+2];
5520           }
5521           else
5522           {
5523             u=1;
5524             gte_u=gte_u_unknown;
5525           }
5526         }
5527       }
5528       else
5529       {
5530         // Internal branch, flag target
5531         bt[(ba[i]-start)>>2]=1;
5532         if(ba[i]<=start+i*4) {
5533           // Backward branch
5534           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5535           {
5536             // Unconditional branch
5537             temp_u=1;
5538             temp_gte_u=0;
5539           } else {
5540             // Conditional branch (not taken case)
5541             temp_u=unneeded_reg[i+2];
5542             temp_gte_u&=gte_unneeded[i+2];
5543           }
5544           // Merge in delay slot
5545           temp_u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5546           temp_u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5547           temp_u|=1;
5548           temp_gte_u|=gte_rt[i+1];
5549           temp_gte_u&=~gte_rs[i+1];
5550           // If branch is "likely" (and conditional)
5551           // then we skip the delay slot on the fall-thru path
5552           if(likely[i]) {
5553             if(i<slen-1) {
5554               temp_u&=unneeded_reg[i+2];
5555               temp_gte_u&=gte_unneeded[i+2];
5556             }
5557             else
5558             {
5559               temp_u=1;
5560               temp_gte_u=gte_u_unknown;
5561             }
5562           }
5563           temp_u|=(1LL<<rt1[i])|(1LL<<rt2[i]);
5564           temp_u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5565           temp_u|=1;
5566           temp_gte_u|=gte_rt[i];
5567           temp_gte_u&=~gte_rs[i];
5568           unneeded_reg[i]=temp_u;
5569           gte_unneeded[i]=temp_gte_u;
5570           // Only go three levels deep.  This recursion can take an
5571           // excessive amount of time if there are a lot of nested loops.
5572           if(r<2) {
5573             unneeded_registers((ba[i]-start)>>2,i-1,r+1);
5574           }else{
5575             unneeded_reg[(ba[i]-start)>>2]=1;
5576             gte_unneeded[(ba[i]-start)>>2]=gte_u_unknown;
5577           }
5578         } /*else*/ if(1) {
5579           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5580           {
5581             // Unconditional branch
5582             u=unneeded_reg[(ba[i]-start)>>2];
5583             gte_u=gte_unneeded[(ba[i]-start)>>2];
5584             branch_unneeded_reg[i]=u;
5585             // Merge in delay slot
5586             u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5587             u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5588             u|=1;
5589             gte_u|=gte_rt[i+1];
5590             gte_u&=~gte_rs[i+1];
5591           } else {
5592             // Conditional branch
5593             b=unneeded_reg[(ba[i]-start)>>2];
5594             gte_b=gte_unneeded[(ba[i]-start)>>2];
5595             branch_unneeded_reg[i]=b;
5596             // Branch delay slot
5597             b|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5598             b&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5599             b|=1;
5600             gte_b|=gte_rt[i+1];
5601             gte_b&=~gte_rs[i+1];
5602             // If branch is "likely" then we skip the
5603             // delay slot on the fall-thru path
5604             if(likely[i]) {
5605               u=b;
5606               gte_u=gte_b;
5607               if(i<slen-1) {
5608                 u&=unneeded_reg[i+2];
5609                 gte_u&=gte_unneeded[i+2];
5610               }
5611             } else {
5612               u&=b;
5613               gte_u&=gte_b;
5614             }
5615             if(i<slen-1) {
5616               branch_unneeded_reg[i]&=unneeded_reg[i+2];
5617             } else {
5618               branch_unneeded_reg[i]=1;
5619             }
5620           }
5621         }
5622       }
5623     }
5624     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5625     {
5626       // SYSCALL instruction (software interrupt)
5627       u=1;
5628     }
5629     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5630     {
5631       // ERET instruction (return from interrupt)
5632       u=1;
5633     }
5634     //u=1; // DEBUG
5635     // Written registers are unneeded
5636     u|=1LL<<rt1[i];
5637     u|=1LL<<rt2[i];
5638     gte_u|=gte_rt[i];
5639     // Accessed registers are needed
5640     u&=~(1LL<<rs1[i]);
5641     u&=~(1LL<<rs2[i]);
5642     gte_u&=~gte_rs[i];
5643     if(gte_rs[i]&&rt1[i]&&(unneeded_reg[i+1]&(1ll<<rt1[i])))
5644       gte_u|=gte_rs[i]&gte_unneeded[i+1]; // MFC2/CFC2 to dead register, unneeded
5645     // Source-target dependencies
5646     // R0 is always unneeded
5647     u|=1;
5648     // Save it
5649     unneeded_reg[i]=u;
5650     gte_unneeded[i]=gte_u;
5651     /*
5652     printf("ur (%d,%d) %x: ",istart,iend,start+i*4);
5653     printf("U:");
5654     int r;
5655     for(r=1;r<=CCREG;r++) {
5656       if((unneeded_reg[i]>>r)&1) {
5657         if(r==HIREG) printf(" HI");
5658         else if(r==LOREG) printf(" LO");
5659         else printf(" r%d",r);
5660       }
5661     }
5662     printf("\n");
5663     */
5664   }
5665 }
5666
5667 // Write back dirty registers as soon as we will no longer modify them,
5668 // so that we don't end up with lots of writes at the branches.
5669 void clean_registers(int istart,int iend,int wr)
5670 {
5671   int i;
5672   int r;
5673   u_int will_dirty_i,will_dirty_next,temp_will_dirty;
5674   u_int wont_dirty_i,wont_dirty_next,temp_wont_dirty;
5675   if(iend==slen-1) {
5676     will_dirty_i=will_dirty_next=0;
5677     wont_dirty_i=wont_dirty_next=0;
5678   }else{
5679     will_dirty_i=will_dirty_next=will_dirty[iend+1];
5680     wont_dirty_i=wont_dirty_next=wont_dirty[iend+1];
5681   }
5682   for (i=iend;i>=istart;i--)
5683   {
5684     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5685     {
5686       if(ba[i]<start || ba[i]>=(start+slen*4))
5687       {
5688         // Branch out of this block, flush all regs
5689         if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5690         {
5691           // Unconditional branch
5692           will_dirty_i=0;
5693           wont_dirty_i=0;
5694           // Merge in delay slot (will dirty)
5695           for(r=0;r<HOST_REGS;r++) {
5696             if(r!=EXCLUDE_REG) {
5697               if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5698               if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5699               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5700               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5701               if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5702               if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5703               if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5704               if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5705               if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5706               if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5707               if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5708               if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5709               if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5710               if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5711             }
5712           }
5713         }
5714         else
5715         {
5716           // Conditional branch
5717           will_dirty_i=0;
5718           wont_dirty_i=wont_dirty_next;
5719           // Merge in delay slot (will dirty)
5720           for(r=0;r<HOST_REGS;r++) {
5721             if(r!=EXCLUDE_REG) {
5722               if(!likely[i]) {
5723                 // Might not dirty if likely branch is not taken
5724                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5725                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5726                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5727                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5728                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5729                 if(branch_regs[i].regmap[r]==0) will_dirty_i&=~(1<<r);
5730                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5731                 //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5732                 //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5733                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5734                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5735                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5736                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5737                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5738               }
5739             }
5740           }
5741         }
5742         // Merge in delay slot (wont dirty)
5743         for(r=0;r<HOST_REGS;r++) {
5744           if(r!=EXCLUDE_REG) {
5745             if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5746             if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5747             if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5748             if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5749             if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5750             if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5751             if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5752             if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5753             if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5754             if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5755           }
5756         }
5757         if(wr) {
5758           #ifndef DESTRUCTIVE_WRITEBACK
5759           branch_regs[i].dirty&=wont_dirty_i;
5760           #endif
5761           branch_regs[i].dirty|=will_dirty_i;
5762         }
5763       }
5764       else
5765       {
5766         // Internal branch
5767         if(ba[i]<=start+i*4) {
5768           // Backward branch
5769           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5770           {
5771             // Unconditional branch
5772             temp_will_dirty=0;
5773             temp_wont_dirty=0;
5774             // Merge in delay slot (will dirty)
5775             for(r=0;r<HOST_REGS;r++) {
5776               if(r!=EXCLUDE_REG) {
5777                 if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5778                 if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5779                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5780                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5781                 if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5782                 if(branch_regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5783                 if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5784                 if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5785                 if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5786                 if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5787                 if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5788                 if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5789                 if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5790                 if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5791               }
5792             }
5793           } else {
5794             // Conditional branch (not taken case)
5795             temp_will_dirty=will_dirty_next;
5796             temp_wont_dirty=wont_dirty_next;
5797             // Merge in delay slot (will dirty)
5798             for(r=0;r<HOST_REGS;r++) {
5799               if(r!=EXCLUDE_REG) {
5800                 if(!likely[i]) {
5801                   // Will not dirty if likely branch is not taken
5802                   if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5803                   if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5804                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5805                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5806                   if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5807                   if(branch_regs[i].regmap[r]==0) temp_will_dirty&=~(1<<r);
5808                   if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5809                   //if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5810                   //if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5811                   if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5812                   if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5813                   if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5814                   if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5815                   if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5816                 }
5817               }
5818             }
5819           }
5820           // Merge in delay slot (wont dirty)
5821           for(r=0;r<HOST_REGS;r++) {
5822             if(r!=EXCLUDE_REG) {
5823               if((regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
5824               if((regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
5825               if((regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
5826               if((regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
5827               if(regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
5828               if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
5829               if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
5830               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
5831               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
5832               if(branch_regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
5833             }
5834           }
5835           // Deal with changed mappings
5836           if(i<iend) {
5837             for(r=0;r<HOST_REGS;r++) {
5838               if(r!=EXCLUDE_REG) {
5839                 if(regs[i].regmap[r]!=regmap_pre[i][r]) {
5840                   temp_will_dirty&=~(1<<r);
5841                   temp_wont_dirty&=~(1<<r);
5842                   if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
5843                     temp_will_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
5844                     temp_wont_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
5845                   } else {
5846                     temp_will_dirty|=1<<r;
5847                     temp_wont_dirty|=1<<r;
5848                   }
5849                 }
5850               }
5851             }
5852           }
5853           if(wr) {
5854             will_dirty[i]=temp_will_dirty;
5855             wont_dirty[i]=temp_wont_dirty;
5856             clean_registers((ba[i]-start)>>2,i-1,0);
5857           }else{
5858             // Limit recursion.  It can take an excessive amount
5859             // of time if there are a lot of nested loops.
5860             will_dirty[(ba[i]-start)>>2]=0;
5861             wont_dirty[(ba[i]-start)>>2]=-1;
5862           }
5863         }
5864         /*else*/ if(1)
5865         {
5866           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5867           {
5868             // Unconditional branch
5869             will_dirty_i=0;
5870             wont_dirty_i=0;
5871           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
5872             for(r=0;r<HOST_REGS;r++) {
5873               if(r!=EXCLUDE_REG) {
5874                 if(branch_regs[i].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5875                   will_dirty_i|=will_dirty[(ba[i]-start)>>2]&(1<<r);
5876                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5877                 }
5878                 if(branch_regs[i].regmap[r]>=0) {
5879                   will_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
5880                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
5881                 }
5882               }
5883             }
5884           //}
5885             // Merge in delay slot
5886             for(r=0;r<HOST_REGS;r++) {
5887               if(r!=EXCLUDE_REG) {
5888                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5889                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5890                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5891                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5892                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5893                 if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5894                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5895                 if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5896                 if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5897                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5898                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5899                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5900                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5901                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5902               }
5903             }
5904           } else {
5905             // Conditional branch
5906             will_dirty_i=will_dirty_next;
5907             wont_dirty_i=wont_dirty_next;
5908           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
5909             for(r=0;r<HOST_REGS;r++) {
5910               if(r!=EXCLUDE_REG) {
5911                 signed char target_reg=branch_regs[i].regmap[r];
5912                 if(target_reg==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5913                   will_dirty_i&=will_dirty[(ba[i]-start)>>2]&(1<<r);
5914                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5915                 }
5916                 else if(target_reg>=0) {
5917                   will_dirty_i&=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
5918                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
5919                 }
5920                 // Treat delay slot as part of branch too
5921                 /*if(regs[i+1].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5922                   will_dirty[i+1]&=will_dirty[(ba[i]-start)>>2]&(1<<r);
5923                   wont_dirty[i+1]|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5924                 }
5925                 else
5926                 {
5927                   will_dirty[i+1]&=~(1<<r);
5928                 }*/
5929               }
5930             }
5931           //}
5932             // Merge in delay slot
5933             for(r=0;r<HOST_REGS;r++) {
5934               if(r!=EXCLUDE_REG) {
5935                 if(!likely[i]) {
5936                   // Might not dirty if likely branch is not taken
5937                   if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5938                   if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5939                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5940                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5941                   if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5942                   if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5943                   if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5944                   //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5945                   //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5946                   if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5947                   if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5948                   if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5949                   if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5950                   if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5951                 }
5952               }
5953             }
5954           }
5955           // Merge in delay slot (won't dirty)
5956           for(r=0;r<HOST_REGS;r++) {
5957             if(r!=EXCLUDE_REG) {
5958               if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5959               if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5960               if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5961               if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5962               if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5963               if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5964               if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5965               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5966               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5967               if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5968             }
5969           }
5970           if(wr) {
5971             #ifndef DESTRUCTIVE_WRITEBACK
5972             branch_regs[i].dirty&=wont_dirty_i;
5973             #endif
5974             branch_regs[i].dirty|=will_dirty_i;
5975           }
5976         }
5977       }
5978     }
5979     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5980     {
5981       // SYSCALL instruction (software interrupt)
5982       will_dirty_i=0;
5983       wont_dirty_i=0;
5984     }
5985     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5986     {
5987       // ERET instruction (return from interrupt)
5988       will_dirty_i=0;
5989       wont_dirty_i=0;
5990     }
5991     will_dirty_next=will_dirty_i;
5992     wont_dirty_next=wont_dirty_i;
5993     for(r=0;r<HOST_REGS;r++) {
5994       if(r!=EXCLUDE_REG) {
5995         if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5996         if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5997         if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5998         if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5999         if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
6000         if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
6001         if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
6002         if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
6003         if(i>istart) {
6004           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP)
6005           {
6006             // Don't store a register immediately after writing it,
6007             // may prevent dual-issue.
6008             if((regs[i].regmap[r]&63)==rt1[i-1]) wont_dirty_i|=1<<r;
6009             if((regs[i].regmap[r]&63)==rt2[i-1]) wont_dirty_i|=1<<r;
6010           }
6011         }
6012       }
6013     }
6014     // Save it
6015     will_dirty[i]=will_dirty_i;
6016     wont_dirty[i]=wont_dirty_i;
6017     // Mark registers that won't be dirtied as not dirty
6018     if(wr) {
6019       /*printf("wr (%d,%d) %x will:",istart,iend,start+i*4);
6020       for(r=0;r<HOST_REGS;r++) {
6021         if((will_dirty_i>>r)&1) {
6022           printf(" r%d",r);
6023         }
6024       }
6025       printf("\n");*/
6026
6027       //if(i==istart||(itype[i-1]!=RJUMP&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP)) {
6028         regs[i].dirty|=will_dirty_i;
6029         #ifndef DESTRUCTIVE_WRITEBACK
6030         regs[i].dirty&=wont_dirty_i;
6031         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
6032         {
6033           if(i<iend-1&&itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
6034             for(r=0;r<HOST_REGS;r++) {
6035               if(r!=EXCLUDE_REG) {
6036                 if(regs[i].regmap[r]==regmap_pre[i+2][r]) {
6037                   regs[i+2].wasdirty&=wont_dirty_i|~(1<<r);
6038                 }else {/*printf("i: %x (%d) mismatch(+2): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6039               }
6040             }
6041           }
6042         }
6043         else
6044         {
6045           if(i<iend) {
6046             for(r=0;r<HOST_REGS;r++) {
6047               if(r!=EXCLUDE_REG) {
6048                 if(regs[i].regmap[r]==regmap_pre[i+1][r]) {
6049                   regs[i+1].wasdirty&=wont_dirty_i|~(1<<r);
6050                 }else {/*printf("i: %x (%d) mismatch(+1): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6051               }
6052             }
6053           }
6054         }
6055         #endif
6056       //}
6057     }
6058     // Deal with changed mappings
6059     temp_will_dirty=will_dirty_i;
6060     temp_wont_dirty=wont_dirty_i;
6061     for(r=0;r<HOST_REGS;r++) {
6062       if(r!=EXCLUDE_REG) {
6063         int nr;
6064         if(regs[i].regmap[r]==regmap_pre[i][r]) {
6065           if(wr) {
6066             #ifndef DESTRUCTIVE_WRITEBACK
6067             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6068             #endif
6069             regs[i].wasdirty|=will_dirty_i&(1<<r);
6070           }
6071         }
6072         else if(regmap_pre[i][r]>=0&&(nr=get_reg(regs[i].regmap,regmap_pre[i][r]))>=0) {
6073           // Register moved to a different register
6074           will_dirty_i&=~(1<<r);
6075           wont_dirty_i&=~(1<<r);
6076           will_dirty_i|=((temp_will_dirty>>nr)&1)<<r;
6077           wont_dirty_i|=((temp_wont_dirty>>nr)&1)<<r;
6078           if(wr) {
6079             #ifndef DESTRUCTIVE_WRITEBACK
6080             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6081             #endif
6082             regs[i].wasdirty|=will_dirty_i&(1<<r);
6083           }
6084         }
6085         else {
6086           will_dirty_i&=~(1<<r);
6087           wont_dirty_i&=~(1<<r);
6088           if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6089             will_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6090             wont_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6091           } else {
6092             wont_dirty_i|=1<<r;
6093             /*printf("i: %x (%d) mismatch: %d\n",start+i*4,i,r);assert(!((will_dirty>>r)&1));*/
6094           }
6095         }
6096       }
6097     }
6098   }
6099 }
6100
6101 #ifdef DISASM
6102   /* disassembly */
6103 void disassemble_inst(int i)
6104 {
6105     if (bt[i]) printf("*"); else printf(" ");
6106     switch(itype[i]) {
6107       case UJUMP:
6108         printf (" %x: %s %8x\n",start+i*4,insn[i],ba[i]);break;
6109       case CJUMP:
6110         printf (" %x: %s r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],i?start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14):*ba);break;
6111       case SJUMP:
6112         printf (" %x: %s r%d,%8x\n",start+i*4,insn[i],rs1[i],start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14));break;
6113       case RJUMP:
6114         if (opcode[i]==0x9&&rt1[i]!=31)
6115           printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i]);
6116         else
6117           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6118         break;
6119       case SPAN:
6120         printf (" %x: %s (pagespan) r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],ba[i]);break;
6121       case IMM16:
6122         if(opcode[i]==0xf) //LUI
6123           printf (" %x: %s r%d,%4x0000\n",start+i*4,insn[i],rt1[i],imm[i]&0xffff);
6124         else
6125           printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6126         break;
6127       case LOAD:
6128       case LOADLR:
6129         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6130         break;
6131       case STORE:
6132       case STORELR:
6133         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rs2[i],rs1[i],imm[i]);
6134         break;
6135       case ALU:
6136       case SHIFT:
6137         printf (" %x: %s r%d,r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i],rs2[i]);
6138         break;
6139       case MULTDIV:
6140         printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rs1[i],rs2[i]);
6141         break;
6142       case SHIFTIMM:
6143         printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6144         break;
6145       case MOV:
6146         if((opcode2[i]&0x1d)==0x10)
6147           printf (" %x: %s r%d\n",start+i*4,insn[i],rt1[i]);
6148         else if((opcode2[i]&0x1d)==0x11)
6149           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6150         else
6151           printf (" %x: %s\n",start+i*4,insn[i]);
6152         break;
6153       case COP0:
6154         if(opcode2[i]==0)
6155           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC0
6156         else if(opcode2[i]==4)
6157           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC0
6158         else printf (" %x: %s\n",start+i*4,insn[i]);
6159         break;
6160       case COP1:
6161         if(opcode2[i]<3)
6162           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC1
6163         else if(opcode2[i]>3)
6164           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC1
6165         else printf (" %x: %s\n",start+i*4,insn[i]);
6166         break;
6167       case COP2:
6168         if(opcode2[i]<3)
6169           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC2
6170         else if(opcode2[i]>3)
6171           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC2
6172         else printf (" %x: %s\n",start+i*4,insn[i]);
6173         break;
6174       case C1LS:
6175         printf (" %x: %s cpr1[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6176         break;
6177       case C2LS:
6178         printf (" %x: %s cpr2[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6179         break;
6180       case INTCALL:
6181         printf (" %x: %s (INTCALL)\n",start+i*4,insn[i]);
6182         break;
6183       default:
6184         //printf (" %s %8x\n",insn[i],source[i]);
6185         printf (" %x: %s\n",start+i*4,insn[i]);
6186     }
6187 }
6188 #else
6189 static void disassemble_inst(int i) {}
6190 #endif // DISASM
6191
6192 #define DRC_TEST_VAL 0x74657374
6193
6194 static void new_dynarec_test(void)
6195 {
6196   int (*testfunc)(void);
6197   void *beginning;
6198   int ret[2];
6199   size_t i;
6200
6201   SysPrintf("testing if we can run recompiled code...\n");
6202   ((volatile u_int *)out)[0]++; // make cache dirty
6203
6204   for (i = 0; i < ARRAY_SIZE(ret); i++) {
6205     out = translation_cache;
6206     beginning = start_block();
6207     emit_movimm(DRC_TEST_VAL + i, 0); // test
6208     emit_ret();
6209     literal_pool(0);
6210     end_block(beginning);
6211     testfunc = beginning;
6212     ret[i] = testfunc();
6213   }
6214
6215   if (ret[0] == DRC_TEST_VAL && ret[1] == DRC_TEST_VAL + 1)
6216     SysPrintf("test passed.\n");
6217   else
6218     SysPrintf("test failed, will likely crash soon (r=%08x %08x)\n", ret[0], ret[1]);
6219   out = translation_cache;
6220 }
6221
6222 // clear the state completely, instead of just marking
6223 // things invalid like invalidate_all_pages() does
6224 void new_dynarec_clear_full()
6225 {
6226   int n;
6227   out = translation_cache;
6228   memset(invalid_code,1,sizeof(invalid_code));
6229   memset(hash_table,0xff,sizeof(hash_table));
6230   memset(mini_ht,-1,sizeof(mini_ht));
6231   memset(restore_candidate,0,sizeof(restore_candidate));
6232   memset(shadow,0,sizeof(shadow));
6233   copy=shadow;
6234   expirep=16384; // Expiry pointer, +2 blocks
6235   pending_exception=0;
6236   literalcount=0;
6237   stop_after_jal=0;
6238   inv_code_start=inv_code_end=~0;
6239   // TLB
6240   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6241   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6242   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6243 }
6244
6245 void new_dynarec_init()
6246 {
6247   SysPrintf("Init new dynarec\n");
6248
6249   // allocate/prepare a buffer for translation cache
6250   // see assem_arm.h for some explanation
6251 #if   defined(BASE_ADDR_FIXED)
6252   if (mmap(translation_cache, 1 << TARGET_SIZE_2,
6253             PROT_READ | PROT_WRITE | PROT_EXEC,
6254             MAP_PRIVATE | MAP_ANONYMOUS,
6255             -1, 0) != translation_cache) {
6256     SysPrintf("mmap() failed: %s\n", strerror(errno));
6257     SysPrintf("disable BASE_ADDR_FIXED and recompile\n");
6258     abort();
6259   }
6260 #elif defined(BASE_ADDR_DYNAMIC)
6261   #ifdef VITA
6262   sceBlock = sceKernelAllocMemBlockForVM("code", 1 << TARGET_SIZE_2);
6263   if (sceBlock < 0)
6264     SysPrintf("sceKernelAllocMemBlockForVM failed\n");
6265   int ret = sceKernelGetMemBlockBase(sceBlock, (void **)&translation_cache);
6266   if (ret < 0)
6267     SysPrintf("sceKernelGetMemBlockBase failed\n");
6268   #else
6269   translation_cache = mmap (NULL, 1 << TARGET_SIZE_2,
6270             PROT_READ | PROT_WRITE | PROT_EXEC,
6271             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
6272   if (translation_cache == MAP_FAILED) {
6273     SysPrintf("mmap() failed: %s\n", strerror(errno));
6274     abort();
6275   }
6276   #endif
6277 #else
6278   #ifndef NO_WRITE_EXEC
6279   // not all systems allow execute in data segment by default
6280   if (mprotect(translation_cache, 1<<TARGET_SIZE_2, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
6281     SysPrintf("mprotect() failed: %s\n", strerror(errno));
6282   #endif
6283 #endif
6284   out = translation_cache;
6285   cycle_multiplier=200;
6286   new_dynarec_clear_full();
6287 #ifdef HOST_IMM8
6288   // Copy this into local area so we don't have to put it in every literal pool
6289   invc_ptr=invalid_code;
6290 #endif
6291   arch_init();
6292   new_dynarec_test();
6293 #ifndef RAM_FIXED
6294   ram_offset=(uintptr_t)rdram-0x80000000;
6295 #endif
6296   if (ram_offset!=0)
6297     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
6298 }
6299
6300 void new_dynarec_cleanup()
6301 {
6302   int n;
6303 #if defined(BASE_ADDR_FIXED) || defined(BASE_ADDR_DYNAMIC)
6304   #ifdef VITA
6305   sceKernelFreeMemBlock(sceBlock);
6306   sceBlock = -1;
6307   #else
6308   if (munmap(translation_cache, 1<<TARGET_SIZE_2) < 0)
6309     SysPrintf("munmap() failed\n");
6310   #endif
6311 #endif
6312   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6313   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6314   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6315   #ifdef ROM_COPY
6316   if (munmap (ROM_COPY, 67108864) < 0) {SysPrintf("munmap() failed\n");}
6317   #endif
6318 }
6319
6320 static u_int *get_source_start(u_int addr, u_int *limit)
6321 {
6322   if (addr < 0x00200000 ||
6323     (0xa0000000 <= addr && addr < 0xa0200000)) {
6324     // used for BIOS calls mostly?
6325     *limit = (addr&0xa0000000)|0x00200000;
6326     return (u_int *)(rdram + (addr&0x1fffff));
6327   }
6328   else if (!Config.HLE && (
6329     /* (0x9fc00000 <= addr && addr < 0x9fc80000) ||*/
6330     (0xbfc00000 <= addr && addr < 0xbfc80000))) {
6331     // BIOS
6332     *limit = (addr & 0xfff00000) | 0x80000;
6333     return (u_int *)((u_char *)psxR + (addr&0x7ffff));
6334   }
6335   else if (addr >= 0x80000000 && addr < 0x80000000+RAM_SIZE) {
6336     *limit = (addr & 0x80600000) + 0x00200000;
6337     return (u_int *)(rdram + (addr&0x1fffff));
6338   }
6339   return NULL;
6340 }
6341
6342 static u_int scan_for_ret(u_int addr)
6343 {
6344   u_int limit = 0;
6345   u_int *mem;
6346
6347   mem = get_source_start(addr, &limit);
6348   if (mem == NULL)
6349     return addr;
6350
6351   if (limit > addr + 0x1000)
6352     limit = addr + 0x1000;
6353   for (; addr < limit; addr += 4, mem++) {
6354     if (*mem == 0x03e00008) // jr $ra
6355       return addr + 8;
6356   }
6357   return addr;
6358 }
6359
6360 struct savestate_block {
6361   uint32_t addr;
6362   uint32_t regflags;
6363 };
6364
6365 static int addr_cmp(const void *p1_, const void *p2_)
6366 {
6367   const struct savestate_block *p1 = p1_, *p2 = p2_;
6368   return p1->addr - p2->addr;
6369 }
6370
6371 int new_dynarec_save_blocks(void *save, int size)
6372 {
6373   struct savestate_block *blocks = save;
6374   int maxcount = size / sizeof(blocks[0]);
6375   struct savestate_block tmp_blocks[1024];
6376   struct ll_entry *head;
6377   int p, s, d, o, bcnt;
6378   u_int addr;
6379
6380   o = 0;
6381   for (p = 0; p < ARRAY_SIZE(jump_in); p++) {
6382     bcnt = 0;
6383     for (head = jump_in[p]; head != NULL; head = head->next) {
6384       tmp_blocks[bcnt].addr = head->vaddr;
6385       tmp_blocks[bcnt].regflags = head->reg_sv_flags;
6386       bcnt++;
6387     }
6388     if (bcnt < 1)
6389       continue;
6390     qsort(tmp_blocks, bcnt, sizeof(tmp_blocks[0]), addr_cmp);
6391
6392     addr = tmp_blocks[0].addr;
6393     for (s = d = 0; s < bcnt; s++) {
6394       if (tmp_blocks[s].addr < addr)
6395         continue;
6396       if (d == 0 || tmp_blocks[d-1].addr != tmp_blocks[s].addr)
6397         tmp_blocks[d++] = tmp_blocks[s];
6398       addr = scan_for_ret(tmp_blocks[s].addr);
6399     }
6400
6401     if (o + d > maxcount)
6402       d = maxcount - o;
6403     memcpy(&blocks[o], tmp_blocks, d * sizeof(blocks[0]));
6404     o += d;
6405   }
6406
6407   return o * sizeof(blocks[0]);
6408 }
6409
6410 void new_dynarec_load_blocks(const void *save, int size)
6411 {
6412   const struct savestate_block *blocks = save;
6413   int count = size / sizeof(blocks[0]);
6414   u_int regs_save[32];
6415   uint32_t f;
6416   int i, b;
6417
6418   get_addr(psxRegs.pc);
6419
6420   // change GPRs for speculation to at least partially work..
6421   memcpy(regs_save, &psxRegs.GPR, sizeof(regs_save));
6422   for (i = 1; i < 32; i++)
6423     psxRegs.GPR.r[i] = 0x80000000;
6424
6425   for (b = 0; b < count; b++) {
6426     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6427       if (f & 1)
6428         psxRegs.GPR.r[i] = 0x1f800000;
6429     }
6430
6431     get_addr(blocks[b].addr);
6432
6433     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6434       if (f & 1)
6435         psxRegs.GPR.r[i] = 0x80000000;
6436     }
6437   }
6438
6439   memcpy(&psxRegs.GPR, regs_save, sizeof(regs_save));
6440 }
6441
6442 int new_recompile_block(int addr)
6443 {
6444   u_int pagelimit = 0;
6445   u_int state_rflags = 0;
6446   int i;
6447
6448   assem_debug("NOTCOMPILED: addr = %x -> %p\n", addr, out);
6449   //printf("TRACE: count=%d next=%d (compile %x)\n",Count,next_interupt,addr);
6450   //if(debug)
6451   //printf("fpu mapping=%x enabled=%x\n",(Status & 0x04000000)>>26,(Status & 0x20000000)>>29);
6452
6453   // this is just for speculation
6454   for (i = 1; i < 32; i++) {
6455     if ((psxRegs.GPR.r[i] & 0xffff0000) == 0x1f800000)
6456       state_rflags |= 1 << i;
6457   }
6458
6459   start = (u_int)addr&~3;
6460   //assert(((u_int)addr&1)==0);
6461   new_dynarec_did_compile=1;
6462   if (Config.HLE && start == 0x80001000) // hlecall
6463   {
6464     // XXX: is this enough? Maybe check hleSoftCall?
6465     void *beginning=start_block();
6466     u_int page=get_page(start);
6467
6468     invalid_code[start>>12]=0;
6469     emit_movimm(start,0);
6470     emit_writeword(0,&pcaddr);
6471     emit_jmp(new_dyna_leave);
6472     literal_pool(0);
6473     end_block(beginning);
6474     ll_add_flags(jump_in+page,start,state_rflags,(void *)beginning);
6475     return 0;
6476   }
6477
6478   source = get_source_start(start, &pagelimit);
6479   if (source == NULL) {
6480     SysPrintf("Compile at bogus memory address: %08x\n", addr);
6481     exit(1);
6482   }
6483
6484   /* Pass 1: disassemble */
6485   /* Pass 2: register dependencies, branch targets */
6486   /* Pass 3: register allocation */
6487   /* Pass 4: branch dependencies */
6488   /* Pass 5: pre-alloc */
6489   /* Pass 6: optimize clean/dirty state */
6490   /* Pass 7: flag 32-bit registers */
6491   /* Pass 8: assembly */
6492   /* Pass 9: linker */
6493   /* Pass 10: garbage collection / free memory */
6494
6495   int j;
6496   int done=0;
6497   unsigned int type,op,op2;
6498
6499   //printf("addr = %x source = %x %x\n", addr,source,source[0]);
6500
6501   /* Pass 1 disassembly */
6502
6503   for(i=0;!done;i++) {
6504     bt[i]=0;likely[i]=0;ooo[i]=0;op2=0;
6505     minimum_free_regs[i]=0;
6506     opcode[i]=op=source[i]>>26;
6507     switch(op)
6508     {
6509       case 0x00: strcpy(insn[i],"special"); type=NI;
6510         op2=source[i]&0x3f;
6511         switch(op2)
6512         {
6513           case 0x00: strcpy(insn[i],"SLL"); type=SHIFTIMM; break;
6514           case 0x02: strcpy(insn[i],"SRL"); type=SHIFTIMM; break;
6515           case 0x03: strcpy(insn[i],"SRA"); type=SHIFTIMM; break;
6516           case 0x04: strcpy(insn[i],"SLLV"); type=SHIFT; break;
6517           case 0x06: strcpy(insn[i],"SRLV"); type=SHIFT; break;
6518           case 0x07: strcpy(insn[i],"SRAV"); type=SHIFT; break;
6519           case 0x08: strcpy(insn[i],"JR"); type=RJUMP; break;
6520           case 0x09: strcpy(insn[i],"JALR"); type=RJUMP; break;
6521           case 0x0C: strcpy(insn[i],"SYSCALL"); type=SYSCALL; break;
6522           case 0x0D: strcpy(insn[i],"BREAK"); type=OTHER; break;
6523           case 0x0F: strcpy(insn[i],"SYNC"); type=OTHER; break;
6524           case 0x10: strcpy(insn[i],"MFHI"); type=MOV; break;
6525           case 0x11: strcpy(insn[i],"MTHI"); type=MOV; break;
6526           case 0x12: strcpy(insn[i],"MFLO"); type=MOV; break;
6527           case 0x13: strcpy(insn[i],"MTLO"); type=MOV; break;
6528           case 0x18: strcpy(insn[i],"MULT"); type=MULTDIV; break;
6529           case 0x19: strcpy(insn[i],"MULTU"); type=MULTDIV; break;
6530           case 0x1A: strcpy(insn[i],"DIV"); type=MULTDIV; break;
6531           case 0x1B: strcpy(insn[i],"DIVU"); type=MULTDIV; break;
6532           case 0x20: strcpy(insn[i],"ADD"); type=ALU; break;
6533           case 0x21: strcpy(insn[i],"ADDU"); type=ALU; break;
6534           case 0x22: strcpy(insn[i],"SUB"); type=ALU; break;
6535           case 0x23: strcpy(insn[i],"SUBU"); type=ALU; break;
6536           case 0x24: strcpy(insn[i],"AND"); type=ALU; break;
6537           case 0x25: strcpy(insn[i],"OR"); type=ALU; break;
6538           case 0x26: strcpy(insn[i],"XOR"); type=ALU; break;
6539           case 0x27: strcpy(insn[i],"NOR"); type=ALU; break;
6540           case 0x2A: strcpy(insn[i],"SLT"); type=ALU; break;
6541           case 0x2B: strcpy(insn[i],"SLTU"); type=ALU; break;
6542           case 0x30: strcpy(insn[i],"TGE"); type=NI; break;
6543           case 0x31: strcpy(insn[i],"TGEU"); type=NI; break;
6544           case 0x32: strcpy(insn[i],"TLT"); type=NI; break;
6545           case 0x33: strcpy(insn[i],"TLTU"); type=NI; break;
6546           case 0x34: strcpy(insn[i],"TEQ"); type=NI; break;
6547           case 0x36: strcpy(insn[i],"TNE"); type=NI; break;
6548 #if 0
6549           case 0x14: strcpy(insn[i],"DSLLV"); type=SHIFT; break;
6550           case 0x16: strcpy(insn[i],"DSRLV"); type=SHIFT; break;
6551           case 0x17: strcpy(insn[i],"DSRAV"); type=SHIFT; break;
6552           case 0x1C: strcpy(insn[i],"DMULT"); type=MULTDIV; break;
6553           case 0x1D: strcpy(insn[i],"DMULTU"); type=MULTDIV; break;
6554           case 0x1E: strcpy(insn[i],"DDIV"); type=MULTDIV; break;
6555           case 0x1F: strcpy(insn[i],"DDIVU"); type=MULTDIV; break;
6556           case 0x2C: strcpy(insn[i],"DADD"); type=ALU; break;
6557           case 0x2D: strcpy(insn[i],"DADDU"); type=ALU; break;
6558           case 0x2E: strcpy(insn[i],"DSUB"); type=ALU; break;
6559           case 0x2F: strcpy(insn[i],"DSUBU"); type=ALU; break;
6560           case 0x38: strcpy(insn[i],"DSLL"); type=SHIFTIMM; break;
6561           case 0x3A: strcpy(insn[i],"DSRL"); type=SHIFTIMM; break;
6562           case 0x3B: strcpy(insn[i],"DSRA"); type=SHIFTIMM; break;
6563           case 0x3C: strcpy(insn[i],"DSLL32"); type=SHIFTIMM; break;
6564           case 0x3E: strcpy(insn[i],"DSRL32"); type=SHIFTIMM; break;
6565           case 0x3F: strcpy(insn[i],"DSRA32"); type=SHIFTIMM; break;
6566 #endif
6567         }
6568         break;
6569       case 0x01: strcpy(insn[i],"regimm"); type=NI;
6570         op2=(source[i]>>16)&0x1f;
6571         switch(op2)
6572         {
6573           case 0x00: strcpy(insn[i],"BLTZ"); type=SJUMP; break;
6574           case 0x01: strcpy(insn[i],"BGEZ"); type=SJUMP; break;
6575           case 0x02: strcpy(insn[i],"BLTZL"); type=SJUMP; break;
6576           case 0x03: strcpy(insn[i],"BGEZL"); type=SJUMP; break;
6577           case 0x08: strcpy(insn[i],"TGEI"); type=NI; break;
6578           case 0x09: strcpy(insn[i],"TGEIU"); type=NI; break;
6579           case 0x0A: strcpy(insn[i],"TLTI"); type=NI; break;
6580           case 0x0B: strcpy(insn[i],"TLTIU"); type=NI; break;
6581           case 0x0C: strcpy(insn[i],"TEQI"); type=NI; break;
6582           case 0x0E: strcpy(insn[i],"TNEI"); type=NI; break;
6583           case 0x10: strcpy(insn[i],"BLTZAL"); type=SJUMP; break;
6584           case 0x11: strcpy(insn[i],"BGEZAL"); type=SJUMP; break;
6585           case 0x12: strcpy(insn[i],"BLTZALL"); type=SJUMP; break;
6586           case 0x13: strcpy(insn[i],"BGEZALL"); type=SJUMP; break;
6587         }
6588         break;
6589       case 0x02: strcpy(insn[i],"J"); type=UJUMP; break;
6590       case 0x03: strcpy(insn[i],"JAL"); type=UJUMP; break;
6591       case 0x04: strcpy(insn[i],"BEQ"); type=CJUMP; break;
6592       case 0x05: strcpy(insn[i],"BNE"); type=CJUMP; break;
6593       case 0x06: strcpy(insn[i],"BLEZ"); type=CJUMP; break;
6594       case 0x07: strcpy(insn[i],"BGTZ"); type=CJUMP; break;
6595       case 0x08: strcpy(insn[i],"ADDI"); type=IMM16; break;
6596       case 0x09: strcpy(insn[i],"ADDIU"); type=IMM16; break;
6597       case 0x0A: strcpy(insn[i],"SLTI"); type=IMM16; break;
6598       case 0x0B: strcpy(insn[i],"SLTIU"); type=IMM16; break;
6599       case 0x0C: strcpy(insn[i],"ANDI"); type=IMM16; break;
6600       case 0x0D: strcpy(insn[i],"ORI"); type=IMM16; break;
6601       case 0x0E: strcpy(insn[i],"XORI"); type=IMM16; break;
6602       case 0x0F: strcpy(insn[i],"LUI"); type=IMM16; break;
6603       case 0x10: strcpy(insn[i],"cop0"); type=NI;
6604         op2=(source[i]>>21)&0x1f;
6605         switch(op2)
6606         {
6607           case 0x00: strcpy(insn[i],"MFC0"); type=COP0; break;
6608           case 0x02: strcpy(insn[i],"CFC0"); type=COP0; break;
6609           case 0x04: strcpy(insn[i],"MTC0"); type=COP0; break;
6610           case 0x06: strcpy(insn[i],"CTC0"); type=COP0; break;
6611           case 0x10: strcpy(insn[i],"RFE"); type=COP0; break;
6612         }
6613         break;
6614       case 0x11: strcpy(insn[i],"cop1"); type=COP1;
6615         op2=(source[i]>>21)&0x1f;
6616         break;
6617 #if 0
6618       case 0x14: strcpy(insn[i],"BEQL"); type=CJUMP; break;
6619       case 0x15: strcpy(insn[i],"BNEL"); type=CJUMP; break;
6620       case 0x16: strcpy(insn[i],"BLEZL"); type=CJUMP; break;
6621       case 0x17: strcpy(insn[i],"BGTZL"); type=CJUMP; break;
6622       case 0x18: strcpy(insn[i],"DADDI"); type=IMM16; break;
6623       case 0x19: strcpy(insn[i],"DADDIU"); type=IMM16; break;
6624       case 0x1A: strcpy(insn[i],"LDL"); type=LOADLR; break;
6625       case 0x1B: strcpy(insn[i],"LDR"); type=LOADLR; break;
6626 #endif
6627       case 0x20: strcpy(insn[i],"LB"); type=LOAD; break;
6628       case 0x21: strcpy(insn[i],"LH"); type=LOAD; break;
6629       case 0x22: strcpy(insn[i],"LWL"); type=LOADLR; break;
6630       case 0x23: strcpy(insn[i],"LW"); type=LOAD; break;
6631       case 0x24: strcpy(insn[i],"LBU"); type=LOAD; break;
6632       case 0x25: strcpy(insn[i],"LHU"); type=LOAD; break;
6633       case 0x26: strcpy(insn[i],"LWR"); type=LOADLR; break;
6634 #if 0
6635       case 0x27: strcpy(insn[i],"LWU"); type=LOAD; break;
6636 #endif
6637       case 0x28: strcpy(insn[i],"SB"); type=STORE; break;
6638       case 0x29: strcpy(insn[i],"SH"); type=STORE; break;
6639       case 0x2A: strcpy(insn[i],"SWL"); type=STORELR; break;
6640       case 0x2B: strcpy(insn[i],"SW"); type=STORE; break;
6641 #if 0
6642       case 0x2C: strcpy(insn[i],"SDL"); type=STORELR; break;
6643       case 0x2D: strcpy(insn[i],"SDR"); type=STORELR; break;
6644 #endif
6645       case 0x2E: strcpy(insn[i],"SWR"); type=STORELR; break;
6646       case 0x2F: strcpy(insn[i],"CACHE"); type=NOP; break;
6647       case 0x30: strcpy(insn[i],"LL"); type=NI; break;
6648       case 0x31: strcpy(insn[i],"LWC1"); type=C1LS; break;
6649 #if 0
6650       case 0x34: strcpy(insn[i],"LLD"); type=NI; break;
6651       case 0x35: strcpy(insn[i],"LDC1"); type=C1LS; break;
6652       case 0x37: strcpy(insn[i],"LD"); type=LOAD; break;
6653 #endif
6654       case 0x38: strcpy(insn[i],"SC"); type=NI; break;
6655       case 0x39: strcpy(insn[i],"SWC1"); type=C1LS; break;
6656 #if 0
6657       case 0x3C: strcpy(insn[i],"SCD"); type=NI; break;
6658       case 0x3D: strcpy(insn[i],"SDC1"); type=C1LS; break;
6659       case 0x3F: strcpy(insn[i],"SD"); type=STORE; break;
6660 #endif
6661       case 0x12: strcpy(insn[i],"COP2"); type=NI;
6662         op2=(source[i]>>21)&0x1f;
6663         //if (op2 & 0x10)
6664         if (source[i]&0x3f) { // use this hack to support old savestates with patched gte insns
6665           if (gte_handlers[source[i]&0x3f]!=NULL) {
6666             if (gte_regnames[source[i]&0x3f]!=NULL)
6667               strcpy(insn[i],gte_regnames[source[i]&0x3f]);
6668             else
6669               snprintf(insn[i], sizeof(insn[i]), "COP2 %x", source[i]&0x3f);
6670             type=C2OP;
6671           }
6672         }
6673         else switch(op2)
6674         {
6675           case 0x00: strcpy(insn[i],"MFC2"); type=COP2; break;
6676           case 0x02: strcpy(insn[i],"CFC2"); type=COP2; break;
6677           case 0x04: strcpy(insn[i],"MTC2"); type=COP2; break;
6678           case 0x06: strcpy(insn[i],"CTC2"); type=COP2; break;
6679         }
6680         break;
6681       case 0x32: strcpy(insn[i],"LWC2"); type=C2LS; break;
6682       case 0x3A: strcpy(insn[i],"SWC2"); type=C2LS; break;
6683       case 0x3B: strcpy(insn[i],"HLECALL"); type=HLECALL; break;
6684       default: strcpy(insn[i],"???"); type=NI;
6685         SysPrintf("NI %08x @%08x (%08x)\n", source[i], addr + i*4, addr);
6686         break;
6687     }
6688     itype[i]=type;
6689     opcode2[i]=op2;
6690     /* Get registers/immediates */
6691     lt1[i]=0;
6692     us1[i]=0;
6693     us2[i]=0;
6694     dep1[i]=0;
6695     dep2[i]=0;
6696     gte_rs[i]=gte_rt[i]=0;
6697     switch(type) {
6698       case LOAD:
6699         rs1[i]=(source[i]>>21)&0x1f;
6700         rs2[i]=0;
6701         rt1[i]=(source[i]>>16)&0x1f;
6702         rt2[i]=0;
6703         imm[i]=(short)source[i];
6704         break;
6705       case STORE:
6706       case STORELR:
6707         rs1[i]=(source[i]>>21)&0x1f;
6708         rs2[i]=(source[i]>>16)&0x1f;
6709         rt1[i]=0;
6710         rt2[i]=0;
6711         imm[i]=(short)source[i];
6712         if(op==0x2c||op==0x2d||op==0x3f) us1[i]=rs2[i]; // 64-bit SDL/SDR/SD
6713         break;
6714       case LOADLR:
6715         // LWL/LWR only load part of the register,
6716         // therefore the target register must be treated as a source too
6717         rs1[i]=(source[i]>>21)&0x1f;
6718         rs2[i]=(source[i]>>16)&0x1f;
6719         rt1[i]=(source[i]>>16)&0x1f;
6720         rt2[i]=0;
6721         imm[i]=(short)source[i];
6722         if(op==0x1a||op==0x1b) us1[i]=rs2[i]; // LDR/LDL
6723         if(op==0x26) dep1[i]=rt1[i]; // LWR
6724         break;
6725       case IMM16:
6726         if (op==0x0f) rs1[i]=0; // LUI instruction has no source register
6727         else rs1[i]=(source[i]>>21)&0x1f;
6728         rs2[i]=0;
6729         rt1[i]=(source[i]>>16)&0x1f;
6730         rt2[i]=0;
6731         if(op>=0x0c&&op<=0x0e) { // ANDI/ORI/XORI
6732           imm[i]=(unsigned short)source[i];
6733         }else{
6734           imm[i]=(short)source[i];
6735         }
6736         if(op==0x18||op==0x19) us1[i]=rs1[i]; // DADDI/DADDIU
6737         if(op==0x0a||op==0x0b) us1[i]=rs1[i]; // SLTI/SLTIU
6738         if(op==0x0d||op==0x0e) dep1[i]=rs1[i]; // ORI/XORI
6739         break;
6740       case UJUMP:
6741         rs1[i]=0;
6742         rs2[i]=0;
6743         rt1[i]=0;
6744         rt2[i]=0;
6745         // The JAL instruction writes to r31.
6746         if (op&1) {
6747           rt1[i]=31;
6748         }
6749         rs2[i]=CCREG;
6750         break;
6751       case RJUMP:
6752         rs1[i]=(source[i]>>21)&0x1f;
6753         rs2[i]=0;
6754         rt1[i]=0;
6755         rt2[i]=0;
6756         // The JALR instruction writes to rd.
6757         if (op2&1) {
6758           rt1[i]=(source[i]>>11)&0x1f;
6759         }
6760         rs2[i]=CCREG;
6761         break;
6762       case CJUMP:
6763         rs1[i]=(source[i]>>21)&0x1f;
6764         rs2[i]=(source[i]>>16)&0x1f;
6765         rt1[i]=0;
6766         rt2[i]=0;
6767         if(op&2) { // BGTZ/BLEZ
6768           rs2[i]=0;
6769         }
6770         us1[i]=rs1[i];
6771         us2[i]=rs2[i];
6772         likely[i]=op>>4;
6773         break;
6774       case SJUMP:
6775         rs1[i]=(source[i]>>21)&0x1f;
6776         rs2[i]=CCREG;
6777         rt1[i]=0;
6778         rt2[i]=0;
6779         us1[i]=rs1[i];
6780         if(op2&0x10) { // BxxAL
6781           rt1[i]=31;
6782           // NOTE: If the branch is not taken, r31 is still overwritten
6783         }
6784         likely[i]=(op2&2)>>1;
6785         break;
6786       case ALU:
6787         rs1[i]=(source[i]>>21)&0x1f; // source
6788         rs2[i]=(source[i]>>16)&0x1f; // subtract amount
6789         rt1[i]=(source[i]>>11)&0x1f; // destination
6790         rt2[i]=0;
6791         if(op2==0x2a||op2==0x2b) { // SLT/SLTU
6792           us1[i]=rs1[i];us2[i]=rs2[i];
6793         }
6794         else if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
6795           dep1[i]=rs1[i];dep2[i]=rs2[i];
6796         }
6797         else if(op2>=0x2c&&op2<=0x2f) { // DADD/DSUB
6798           dep1[i]=rs1[i];dep2[i]=rs2[i];
6799         }
6800         break;
6801       case MULTDIV:
6802         rs1[i]=(source[i]>>21)&0x1f; // source
6803         rs2[i]=(source[i]>>16)&0x1f; // divisor
6804         rt1[i]=HIREG;
6805         rt2[i]=LOREG;
6806         if (op2>=0x1c&&op2<=0x1f) { // DMULT/DMULTU/DDIV/DDIVU
6807           us1[i]=rs1[i];us2[i]=rs2[i];
6808         }
6809         break;
6810       case MOV:
6811         rs1[i]=0;
6812         rs2[i]=0;
6813         rt1[i]=0;
6814         rt2[i]=0;
6815         if(op2==0x10) rs1[i]=HIREG; // MFHI
6816         if(op2==0x11) rt1[i]=HIREG; // MTHI
6817         if(op2==0x12) rs1[i]=LOREG; // MFLO
6818         if(op2==0x13) rt1[i]=LOREG; // MTLO
6819         if((op2&0x1d)==0x10) rt1[i]=(source[i]>>11)&0x1f; // MFxx
6820         if((op2&0x1d)==0x11) rs1[i]=(source[i]>>21)&0x1f; // MTxx
6821         dep1[i]=rs1[i];
6822         break;
6823       case SHIFT:
6824         rs1[i]=(source[i]>>16)&0x1f; // target of shift
6825         rs2[i]=(source[i]>>21)&0x1f; // shift amount
6826         rt1[i]=(source[i]>>11)&0x1f; // destination
6827         rt2[i]=0;
6828         // DSLLV/DSRLV/DSRAV are 64-bit
6829         if(op2>=0x14&&op2<=0x17) us1[i]=rs1[i];
6830         break;
6831       case SHIFTIMM:
6832         rs1[i]=(source[i]>>16)&0x1f;
6833         rs2[i]=0;
6834         rt1[i]=(source[i]>>11)&0x1f;
6835         rt2[i]=0;
6836         imm[i]=(source[i]>>6)&0x1f;
6837         // DSxx32 instructions
6838         if(op2>=0x3c) imm[i]|=0x20;
6839         // DSLL/DSRL/DSRA/DSRA32/DSRL32 but not DSLL32 require 64-bit source
6840         if(op2>=0x38&&op2!=0x3c) us1[i]=rs1[i];
6841         break;
6842       case COP0:
6843         rs1[i]=0;
6844         rs2[i]=0;
6845         rt1[i]=0;
6846         rt2[i]=0;
6847         if(op2==0||op2==2) rt1[i]=(source[i]>>16)&0x1F; // MFC0/CFC0
6848         if(op2==4||op2==6) rs1[i]=(source[i]>>16)&0x1F; // MTC0/CTC0
6849         if(op2==4&&((source[i]>>11)&0x1f)==12) rt2[i]=CSREG; // Status
6850         if(op2==16) if((source[i]&0x3f)==0x18) rs2[i]=CCREG; // ERET
6851         break;
6852       case COP1:
6853         rs1[i]=0;
6854         rs2[i]=0;
6855         rt1[i]=0;
6856         rt2[i]=0;
6857         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC1/DMFC1/CFC1
6858         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC1/DMTC1/CTC1
6859         if(op2==5) us1[i]=rs1[i]; // DMTC1
6860         rs2[i]=CSREG;
6861         break;
6862       case COP2:
6863         rs1[i]=0;
6864         rs2[i]=0;
6865         rt1[i]=0;
6866         rt2[i]=0;
6867         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC2/CFC2
6868         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC2/CTC2
6869         rs2[i]=CSREG;
6870         int gr=(source[i]>>11)&0x1F;
6871         switch(op2)
6872         {
6873           case 0x00: gte_rs[i]=1ll<<gr; break; // MFC2
6874           case 0x04: gte_rt[i]=1ll<<gr; break; // MTC2
6875           case 0x02: gte_rs[i]=1ll<<(gr+32); break; // CFC2
6876           case 0x06: gte_rt[i]=1ll<<(gr+32); break; // CTC2
6877         }
6878         break;
6879       case C1LS:
6880         rs1[i]=(source[i]>>21)&0x1F;
6881         rs2[i]=CSREG;
6882         rt1[i]=0;
6883         rt2[i]=0;
6884         imm[i]=(short)source[i];
6885         break;
6886       case C2LS:
6887         rs1[i]=(source[i]>>21)&0x1F;
6888         rs2[i]=0;
6889         rt1[i]=0;
6890         rt2[i]=0;
6891         imm[i]=(short)source[i];
6892         if(op==0x32) gte_rt[i]=1ll<<((source[i]>>16)&0x1F); // LWC2
6893         else gte_rs[i]=1ll<<((source[i]>>16)&0x1F); // SWC2
6894         break;
6895       case C2OP:
6896         rs1[i]=0;
6897         rs2[i]=0;
6898         rt1[i]=0;
6899         rt2[i]=0;
6900         gte_rs[i]=gte_reg_reads[source[i]&0x3f];
6901         gte_rt[i]=gte_reg_writes[source[i]&0x3f];
6902         gte_rt[i]|=1ll<<63; // every op changes flags
6903         if((source[i]&0x3f)==GTE_MVMVA) {
6904           int v = (source[i] >> 15) & 3;
6905           gte_rs[i]&=~0xe3fll;
6906           if(v==3) gte_rs[i]|=0xe00ll;
6907           else gte_rs[i]|=3ll<<(v*2);
6908         }
6909         break;
6910       case SYSCALL:
6911       case HLECALL:
6912       case INTCALL:
6913         rs1[i]=CCREG;
6914         rs2[i]=0;
6915         rt1[i]=0;
6916         rt2[i]=0;
6917         break;
6918       default:
6919         rs1[i]=0;
6920         rs2[i]=0;
6921         rt1[i]=0;
6922         rt2[i]=0;
6923     }
6924     /* Calculate branch target addresses */
6925     if(type==UJUMP)
6926       ba[i]=((start+i*4+4)&0xF0000000)|(((unsigned int)source[i]<<6)>>4);
6927     else if(type==CJUMP&&rs1[i]==rs2[i]&&(op&1))
6928       ba[i]=start+i*4+8; // Ignore never taken branch
6929     else if(type==SJUMP&&rs1[i]==0&&!(op2&1))
6930       ba[i]=start+i*4+8; // Ignore never taken branch
6931     else if(type==CJUMP||type==SJUMP)
6932       ba[i]=start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14);
6933     else ba[i]=-1;
6934     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
6935       int do_in_intrp=0;
6936       // branch in delay slot?
6937       if(type==RJUMP||type==UJUMP||type==CJUMP||type==SJUMP) {
6938         // don't handle first branch and call interpreter if it's hit
6939         SysPrintf("branch in delay slot @%08x (%08x)\n", addr + i*4, addr);
6940         do_in_intrp=1;
6941       }
6942       // basic load delay detection
6943       else if((type==LOAD||type==LOADLR||type==COP0||type==COP2||type==C2LS)&&rt1[i]!=0) {
6944         int t=(ba[i-1]-start)/4;
6945         if(0 <= t && t < i &&(rt1[i]==rs1[t]||rt1[i]==rs2[t])&&itype[t]!=CJUMP&&itype[t]!=SJUMP) {
6946           // jump target wants DS result - potential load delay effect
6947           SysPrintf("load delay @%08x (%08x)\n", addr + i*4, addr);
6948           do_in_intrp=1;
6949           bt[t+1]=1; // expected return from interpreter
6950         }
6951         else if(i>=2&&rt1[i-2]==2&&rt1[i]==2&&rs1[i]!=2&&rs2[i]!=2&&rs1[i-1]!=2&&rs2[i-1]!=2&&
6952               !(i>=3&&(itype[i-3]==RJUMP||itype[i-3]==UJUMP||itype[i-3]==CJUMP||itype[i-3]==SJUMP))) {
6953           // v0 overwrite like this is a sign of trouble, bail out
6954           SysPrintf("v0 overwrite @%08x (%08x)\n", addr + i*4, addr);
6955           do_in_intrp=1;
6956         }
6957       }
6958       if(do_in_intrp) {
6959         rs1[i-1]=CCREG;
6960         rs2[i-1]=rt1[i-1]=rt2[i-1]=0;
6961         ba[i-1]=-1;
6962         itype[i-1]=INTCALL;
6963         done=2;
6964         i--; // don't compile the DS
6965       }
6966     }
6967     /* Is this the end of the block? */
6968     if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)) {
6969       if(rt1[i-1]==0) { // Continue past subroutine call (JAL)
6970         done=2;
6971       }
6972       else {
6973         if(stop_after_jal) done=1;
6974         // Stop on BREAK
6975         if((source[i+1]&0xfc00003f)==0x0d) done=1;
6976       }
6977       // Don't recompile stuff that's already compiled
6978       if(check_addr(start+i*4+4)) done=1;
6979       // Don't get too close to the limit
6980       if(i>MAXBLOCK/2) done=1;
6981     }
6982     if(itype[i]==SYSCALL&&stop_after_jal) done=1;
6983     if(itype[i]==HLECALL||itype[i]==INTCALL) done=2;
6984     if(done==2) {
6985       // Does the block continue due to a branch?
6986       for(j=i-1;j>=0;j--)
6987       {
6988         if(ba[j]==start+i*4) done=j=0; // Branch into delay slot
6989         if(ba[j]==start+i*4+4) done=j=0;
6990         if(ba[j]==start+i*4+8) done=j=0;
6991       }
6992     }
6993     //assert(i<MAXBLOCK-1);
6994     if(start+i*4==pagelimit-4) done=1;
6995     assert(start+i*4<pagelimit);
6996     if (i==MAXBLOCK-1) done=1;
6997     // Stop if we're compiling junk
6998     if(itype[i]==NI&&opcode[i]==0x11) {
6999       done=stop_after_jal=1;
7000       SysPrintf("Disabled speculative precompilation\n");
7001     }
7002   }
7003   slen=i;
7004   if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==RJUMP) {
7005     if(start+i*4==pagelimit) {
7006       itype[i-1]=SPAN;
7007     }
7008   }
7009   assert(slen>0);
7010
7011   /* Pass 2 - Register dependencies and branch targets */
7012
7013   unneeded_registers(0,slen-1,0);
7014
7015   /* Pass 3 - Register allocation */
7016
7017   struct regstat current; // Current register allocations/status
7018   current.dirty=0;
7019   current.u=unneeded_reg[0];
7020   clear_all_regs(current.regmap);
7021   alloc_reg(&current,0,CCREG);
7022   dirty_reg(&current,CCREG);
7023   current.isconst=0;
7024   current.wasconst=0;
7025   current.waswritten=0;
7026   int ds=0;
7027   int cc=0;
7028   int hr=-1;
7029
7030   if((u_int)addr&1) {
7031     // First instruction is delay slot
7032     cc=-1;
7033     bt[1]=1;
7034     ds=1;
7035     unneeded_reg[0]=1;
7036     current.regmap[HOST_BTREG]=BTREG;
7037   }
7038
7039   for(i=0;i<slen;i++)
7040   {
7041     if(bt[i])
7042     {
7043       int hr;
7044       for(hr=0;hr<HOST_REGS;hr++)
7045       {
7046         // Is this really necessary?
7047         if(current.regmap[hr]==0) current.regmap[hr]=-1;
7048       }
7049       current.isconst=0;
7050       current.waswritten=0;
7051     }
7052     if(i>1)
7053     {
7054       if((opcode[i-2]&0x2f)==0x05) // BNE/BNEL
7055       {
7056         if(rs1[i-2]==0||rs2[i-2]==0)
7057         {
7058           if(rs1[i-2]) {
7059             int hr=get_reg(current.regmap,rs1[i-2]|64);
7060             if(hr>=0) current.regmap[hr]=-1;
7061           }
7062           if(rs2[i-2]) {
7063             int hr=get_reg(current.regmap,rs2[i-2]|64);
7064             if(hr>=0) current.regmap[hr]=-1;
7065           }
7066         }
7067       }
7068     }
7069
7070     memcpy(regmap_pre[i],current.regmap,sizeof(current.regmap));
7071     regs[i].wasconst=current.isconst;
7072     regs[i].wasdirty=current.dirty;
7073     regs[i].loadedconst=0;
7074     if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP) {
7075       if(i+1<slen) {
7076         current.u=unneeded_reg[i+1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7077         current.u|=1;
7078       } else {
7079         current.u=1;
7080       }
7081     } else {
7082       if(i+1<slen) {
7083         current.u=branch_unneeded_reg[i]&~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
7084         current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7085         current.u|=1;
7086       } else { SysPrintf("oops, branch at end of block with no delay slot\n");exit(1); }
7087     }
7088     is_ds[i]=ds;
7089     if(ds) {
7090       ds=0; // Skip delay slot, already allocated as part of branch
7091       // ...but we need to alloc it in case something jumps here
7092       if(i+1<slen) {
7093         current.u=branch_unneeded_reg[i-1]&unneeded_reg[i+1];
7094       }else{
7095         current.u=branch_unneeded_reg[i-1];
7096       }
7097       current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7098       current.u|=1;
7099       struct regstat temp;
7100       memcpy(&temp,&current,sizeof(current));
7101       temp.wasdirty=temp.dirty;
7102       // TODO: Take into account unconditional branches, as below
7103       delayslot_alloc(&temp,i);
7104       memcpy(regs[i].regmap,temp.regmap,sizeof(temp.regmap));
7105       regs[i].wasdirty=temp.wasdirty;
7106       regs[i].dirty=temp.dirty;
7107       regs[i].isconst=0;
7108       regs[i].wasconst=0;
7109       current.isconst=0;
7110       // Create entry (branch target) regmap
7111       for(hr=0;hr<HOST_REGS;hr++)
7112       {
7113         int r=temp.regmap[hr];
7114         if(r>=0) {
7115           if(r!=regmap_pre[i][hr]) {
7116             regs[i].regmap_entry[hr]=-1;
7117           }
7118           else
7119           {
7120             if(r<64){
7121               if((current.u>>r)&1) {
7122                 regs[i].regmap_entry[hr]=-1;
7123                 regs[i].regmap[hr]=-1;
7124                 //Don't clear regs in the delay slot as the branch might need them
7125                 //current.regmap[hr]=-1;
7126               }else
7127                 regs[i].regmap_entry[hr]=r;
7128             }
7129             else {
7130               assert(0);
7131             }
7132           }
7133         } else {
7134           // First instruction expects CCREG to be allocated
7135           if(i==0&&hr==HOST_CCREG)
7136             regs[i].regmap_entry[hr]=CCREG;
7137           else
7138             regs[i].regmap_entry[hr]=-1;
7139         }
7140       }
7141     }
7142     else { // Not delay slot
7143       switch(itype[i]) {
7144         case UJUMP:
7145           //current.isconst=0; // DEBUG
7146           //current.wasconst=0; // DEBUG
7147           //regs[i].wasconst=0; // DEBUG
7148           clear_const(&current,rt1[i]);
7149           alloc_cc(&current,i);
7150           dirty_reg(&current,CCREG);
7151           if (rt1[i]==31) {
7152             alloc_reg(&current,i,31);
7153             dirty_reg(&current,31);
7154             //assert(rs1[i+1]!=31&&rs2[i+1]!=31);
7155             //assert(rt1[i+1]!=rt1[i]);
7156             #ifdef REG_PREFETCH
7157             alloc_reg(&current,i,PTEMP);
7158             #endif
7159           }
7160           ooo[i]=1;
7161           delayslot_alloc(&current,i+1);
7162           //current.isconst=0; // DEBUG
7163           ds=1;
7164           //printf("i=%d, isconst=%x\n",i,current.isconst);
7165           break;
7166         case RJUMP:
7167           //current.isconst=0;
7168           //current.wasconst=0;
7169           //regs[i].wasconst=0;
7170           clear_const(&current,rs1[i]);
7171           clear_const(&current,rt1[i]);
7172           alloc_cc(&current,i);
7173           dirty_reg(&current,CCREG);
7174           if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
7175             alloc_reg(&current,i,rs1[i]);
7176             if (rt1[i]!=0) {
7177               alloc_reg(&current,i,rt1[i]);
7178               dirty_reg(&current,rt1[i]);
7179               assert(rs1[i+1]!=rt1[i]&&rs2[i+1]!=rt1[i]);
7180               assert(rt1[i+1]!=rt1[i]);
7181               #ifdef REG_PREFETCH
7182               alloc_reg(&current,i,PTEMP);
7183               #endif
7184             }
7185             #ifdef USE_MINI_HT
7186             if(rs1[i]==31) { // JALR
7187               alloc_reg(&current,i,RHASH);
7188               alloc_reg(&current,i,RHTBL);
7189             }
7190             #endif
7191             delayslot_alloc(&current,i+1);
7192           } else {
7193             // The delay slot overwrites our source register,
7194             // allocate a temporary register to hold the old value.
7195             current.isconst=0;
7196             current.wasconst=0;
7197             regs[i].wasconst=0;
7198             delayslot_alloc(&current,i+1);
7199             current.isconst=0;
7200             alloc_reg(&current,i,RTEMP);
7201           }
7202           //current.isconst=0; // DEBUG
7203           ooo[i]=1;
7204           ds=1;
7205           break;
7206         case CJUMP:
7207           //current.isconst=0;
7208           //current.wasconst=0;
7209           //regs[i].wasconst=0;
7210           clear_const(&current,rs1[i]);
7211           clear_const(&current,rs2[i]);
7212           if((opcode[i]&0x3E)==4) // BEQ/BNE
7213           {
7214             alloc_cc(&current,i);
7215             dirty_reg(&current,CCREG);
7216             if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7217             if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7218             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]))||
7219                (rs2[i]&&(rs2[i]==rt1[i+1]||rs2[i]==rt2[i+1]))) {
7220               // The delay slot overwrites one of our conditions.
7221               // Allocate the branch condition registers instead.
7222               current.isconst=0;
7223               current.wasconst=0;
7224               regs[i].wasconst=0;
7225               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7226               if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7227             }
7228             else
7229             {
7230               ooo[i]=1;
7231               delayslot_alloc(&current,i+1);
7232             }
7233           }
7234           else
7235           if((opcode[i]&0x3E)==6) // BLEZ/BGTZ
7236           {
7237             alloc_cc(&current,i);
7238             dirty_reg(&current,CCREG);
7239             alloc_reg(&current,i,rs1[i]);
7240             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
7241               // The delay slot overwrites one of our conditions.
7242               // Allocate the branch condition registers instead.
7243               current.isconst=0;
7244               current.wasconst=0;
7245               regs[i].wasconst=0;
7246               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7247             }
7248             else
7249             {
7250               ooo[i]=1;
7251               delayslot_alloc(&current,i+1);
7252             }
7253           }
7254           else
7255           // Don't alloc the delay slot yet because we might not execute it
7256           if((opcode[i]&0x3E)==0x14) // BEQL/BNEL
7257           {
7258             current.isconst=0;
7259             current.wasconst=0;
7260             regs[i].wasconst=0;
7261             alloc_cc(&current,i);
7262             dirty_reg(&current,CCREG);
7263             alloc_reg(&current,i,rs1[i]);
7264             alloc_reg(&current,i,rs2[i]);
7265           }
7266           else
7267           if((opcode[i]&0x3E)==0x16) // BLEZL/BGTZL
7268           {
7269             current.isconst=0;
7270             current.wasconst=0;
7271             regs[i].wasconst=0;
7272             alloc_cc(&current,i);
7273             dirty_reg(&current,CCREG);
7274             alloc_reg(&current,i,rs1[i]);
7275           }
7276           ds=1;
7277           //current.isconst=0;
7278           break;
7279         case SJUMP:
7280           //current.isconst=0;
7281           //current.wasconst=0;
7282           //regs[i].wasconst=0;
7283           clear_const(&current,rs1[i]);
7284           clear_const(&current,rt1[i]);
7285           //if((opcode2[i]&0x1E)==0x0) // BLTZ/BGEZ
7286           if((opcode2[i]&0x0E)==0x0) // BLTZ/BGEZ
7287           {
7288             alloc_cc(&current,i);
7289             dirty_reg(&current,CCREG);
7290             alloc_reg(&current,i,rs1[i]);
7291             if (rt1[i]==31) { // BLTZAL/BGEZAL
7292               alloc_reg(&current,i,31);
7293               dirty_reg(&current,31);
7294               //#ifdef REG_PREFETCH
7295               //alloc_reg(&current,i,PTEMP);
7296               //#endif
7297             }
7298             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) // The delay slot overwrites the branch condition.
7299                ||(rt1[i]==31&&(rs1[i+1]==31||rs2[i+1]==31||rt1[i+1]==31||rt2[i+1]==31))) { // DS touches $ra
7300               // Allocate the branch condition registers instead.
7301               current.isconst=0;
7302               current.wasconst=0;
7303               regs[i].wasconst=0;
7304               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7305             }
7306             else
7307             {
7308               ooo[i]=1;
7309               delayslot_alloc(&current,i+1);
7310             }
7311           }
7312           else
7313           // Don't alloc the delay slot yet because we might not execute it
7314           if((opcode2[i]&0x1E)==0x2) // BLTZL/BGEZL
7315           {
7316             current.isconst=0;
7317             current.wasconst=0;
7318             regs[i].wasconst=0;
7319             alloc_cc(&current,i);
7320             dirty_reg(&current,CCREG);
7321             alloc_reg(&current,i,rs1[i]);
7322           }
7323           ds=1;
7324           //current.isconst=0;
7325           break;
7326         case IMM16:
7327           imm16_alloc(&current,i);
7328           break;
7329         case LOAD:
7330         case LOADLR:
7331           load_alloc(&current,i);
7332           break;
7333         case STORE:
7334         case STORELR:
7335           store_alloc(&current,i);
7336           break;
7337         case ALU:
7338           alu_alloc(&current,i);
7339           break;
7340         case SHIFT:
7341           shift_alloc(&current,i);
7342           break;
7343         case MULTDIV:
7344           multdiv_alloc(&current,i);
7345           break;
7346         case SHIFTIMM:
7347           shiftimm_alloc(&current,i);
7348           break;
7349         case MOV:
7350           mov_alloc(&current,i);
7351           break;
7352         case COP0:
7353           cop0_alloc(&current,i);
7354           break;
7355         case COP1:
7356         case COP2:
7357           cop12_alloc(&current,i);
7358           break;
7359         case C1LS:
7360           c1ls_alloc(&current,i);
7361           break;
7362         case C2LS:
7363           c2ls_alloc(&current,i);
7364           break;
7365         case C2OP:
7366           c2op_alloc(&current,i);
7367           break;
7368         case SYSCALL:
7369         case HLECALL:
7370         case INTCALL:
7371           syscall_alloc(&current,i);
7372           break;
7373         case SPAN:
7374           pagespan_alloc(&current,i);
7375           break;
7376       }
7377
7378       // Create entry (branch target) regmap
7379       for(hr=0;hr<HOST_REGS;hr++)
7380       {
7381         int r,or;
7382         r=current.regmap[hr];
7383         if(r>=0) {
7384           if(r!=regmap_pre[i][hr]) {
7385             // TODO: delay slot (?)
7386             or=get_reg(regmap_pre[i],r); // Get old mapping for this register
7387             if(or<0||(r&63)>=TEMPREG){
7388               regs[i].regmap_entry[hr]=-1;
7389             }
7390             else
7391             {
7392               // Just move it to a different register
7393               regs[i].regmap_entry[hr]=r;
7394               // If it was dirty before, it's still dirty
7395               if((regs[i].wasdirty>>or)&1) dirty_reg(&current,r&63);
7396             }
7397           }
7398           else
7399           {
7400             // Unneeded
7401             if(r==0){
7402               regs[i].regmap_entry[hr]=0;
7403             }
7404             else
7405             if(r<64){
7406               if((current.u>>r)&1) {
7407                 regs[i].regmap_entry[hr]=-1;
7408                 //regs[i].regmap[hr]=-1;
7409                 current.regmap[hr]=-1;
7410               }else
7411                 regs[i].regmap_entry[hr]=r;
7412             }
7413             else {
7414               assert(0);
7415             }
7416           }
7417         } else {
7418           // Branches expect CCREG to be allocated at the target
7419           if(regmap_pre[i][hr]==CCREG)
7420             regs[i].regmap_entry[hr]=CCREG;
7421           else
7422             regs[i].regmap_entry[hr]=-1;
7423         }
7424       }
7425       memcpy(regs[i].regmap,current.regmap,sizeof(current.regmap));
7426     }
7427
7428     if(i>0&&(itype[i-1]==STORE||itype[i-1]==STORELR||(itype[i-1]==C2LS&&opcode[i-1]==0x3a))&&(u_int)imm[i-1]<0x800)
7429       current.waswritten|=1<<rs1[i-1];
7430     current.waswritten&=~(1<<rt1[i]);
7431     current.waswritten&=~(1<<rt2[i]);
7432     if((itype[i]==STORE||itype[i]==STORELR||(itype[i]==C2LS&&opcode[i]==0x3a))&&(u_int)imm[i]>=0x800)
7433       current.waswritten&=~(1<<rs1[i]);
7434
7435     /* Branch post-alloc */
7436     if(i>0)
7437     {
7438       current.wasdirty=current.dirty;
7439       switch(itype[i-1]) {
7440         case UJUMP:
7441           memcpy(&branch_regs[i-1],&current,sizeof(current));
7442           branch_regs[i-1].isconst=0;
7443           branch_regs[i-1].wasconst=0;
7444           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7445           alloc_cc(&branch_regs[i-1],i-1);
7446           dirty_reg(&branch_regs[i-1],CCREG);
7447           if(rt1[i-1]==31) { // JAL
7448             alloc_reg(&branch_regs[i-1],i-1,31);
7449             dirty_reg(&branch_regs[i-1],31);
7450           }
7451           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7452           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7453           break;
7454         case RJUMP:
7455           memcpy(&branch_regs[i-1],&current,sizeof(current));
7456           branch_regs[i-1].isconst=0;
7457           branch_regs[i-1].wasconst=0;
7458           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7459           alloc_cc(&branch_regs[i-1],i-1);
7460           dirty_reg(&branch_regs[i-1],CCREG);
7461           alloc_reg(&branch_regs[i-1],i-1,rs1[i-1]);
7462           if(rt1[i-1]!=0) { // JALR
7463             alloc_reg(&branch_regs[i-1],i-1,rt1[i-1]);
7464             dirty_reg(&branch_regs[i-1],rt1[i-1]);
7465           }
7466           #ifdef USE_MINI_HT
7467           if(rs1[i-1]==31) { // JALR
7468             alloc_reg(&branch_regs[i-1],i-1,RHASH);
7469             alloc_reg(&branch_regs[i-1],i-1,RHTBL);
7470           }
7471           #endif
7472           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7473           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7474           break;
7475         case CJUMP:
7476           if((opcode[i-1]&0x3E)==4) // BEQ/BNE
7477           {
7478             alloc_cc(&current,i-1);
7479             dirty_reg(&current,CCREG);
7480             if((rs1[i-1]&&(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]))||
7481                (rs2[i-1]&&(rs2[i-1]==rt1[i]||rs2[i-1]==rt2[i]))) {
7482               // The delay slot overwrote one of our conditions
7483               // Delay slot goes after the test (in order)
7484               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7485               current.u|=1;
7486               delayslot_alloc(&current,i);
7487               current.isconst=0;
7488             }
7489             else
7490             {
7491               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7492               // Alloc the branch condition registers
7493               if(rs1[i-1]) alloc_reg(&current,i-1,rs1[i-1]);
7494               if(rs2[i-1]) alloc_reg(&current,i-1,rs2[i-1]);
7495             }
7496             memcpy(&branch_regs[i-1],&current,sizeof(current));
7497             branch_regs[i-1].isconst=0;
7498             branch_regs[i-1].wasconst=0;
7499             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7500             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7501           }
7502           else
7503           if((opcode[i-1]&0x3E)==6) // BLEZ/BGTZ
7504           {
7505             alloc_cc(&current,i-1);
7506             dirty_reg(&current,CCREG);
7507             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7508               // The delay slot overwrote the branch condition
7509               // Delay slot goes after the test (in order)
7510               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7511               current.u|=1;
7512               delayslot_alloc(&current,i);
7513               current.isconst=0;
7514             }
7515             else
7516             {
7517               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7518               // Alloc the branch condition register
7519               alloc_reg(&current,i-1,rs1[i-1]);
7520             }
7521             memcpy(&branch_regs[i-1],&current,sizeof(current));
7522             branch_regs[i-1].isconst=0;
7523             branch_regs[i-1].wasconst=0;
7524             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7525             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7526           }
7527           else
7528           // Alloc the delay slot in case the branch is taken
7529           if((opcode[i-1]&0x3E)==0x14) // BEQL/BNEL
7530           {
7531             memcpy(&branch_regs[i-1],&current,sizeof(current));
7532             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7533             alloc_cc(&branch_regs[i-1],i);
7534             dirty_reg(&branch_regs[i-1],CCREG);
7535             delayslot_alloc(&branch_regs[i-1],i);
7536             branch_regs[i-1].isconst=0;
7537             alloc_reg(&current,i,CCREG); // Not taken path
7538             dirty_reg(&current,CCREG);
7539             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7540           }
7541           else
7542           if((opcode[i-1]&0x3E)==0x16) // BLEZL/BGTZL
7543           {
7544             memcpy(&branch_regs[i-1],&current,sizeof(current));
7545             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7546             alloc_cc(&branch_regs[i-1],i);
7547             dirty_reg(&branch_regs[i-1],CCREG);
7548             delayslot_alloc(&branch_regs[i-1],i);
7549             branch_regs[i-1].isconst=0;
7550             alloc_reg(&current,i,CCREG); // Not taken path
7551             dirty_reg(&current,CCREG);
7552             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7553           }
7554           break;
7555         case SJUMP:
7556           //if((opcode2[i-1]&0x1E)==0) // BLTZ/BGEZ
7557           if((opcode2[i-1]&0x0E)==0) // BLTZ/BGEZ
7558           {
7559             alloc_cc(&current,i-1);
7560             dirty_reg(&current,CCREG);
7561             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7562               // The delay slot overwrote the branch condition
7563               // Delay slot goes after the test (in order)
7564               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7565               current.u|=1;
7566               delayslot_alloc(&current,i);
7567               current.isconst=0;
7568             }
7569             else
7570             {
7571               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7572               // Alloc the branch condition register
7573               alloc_reg(&current,i-1,rs1[i-1]);
7574             }
7575             memcpy(&branch_regs[i-1],&current,sizeof(current));
7576             branch_regs[i-1].isconst=0;
7577             branch_regs[i-1].wasconst=0;
7578             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7579             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7580           }
7581           else
7582           // Alloc the delay slot in case the branch is taken
7583           if((opcode2[i-1]&0x1E)==2) // BLTZL/BGEZL
7584           {
7585             memcpy(&branch_regs[i-1],&current,sizeof(current));
7586             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7587             alloc_cc(&branch_regs[i-1],i);
7588             dirty_reg(&branch_regs[i-1],CCREG);
7589             delayslot_alloc(&branch_regs[i-1],i);
7590             branch_regs[i-1].isconst=0;
7591             alloc_reg(&current,i,CCREG); // Not taken path
7592             dirty_reg(&current,CCREG);
7593             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7594           }
7595           // FIXME: BLTZAL/BGEZAL
7596           if(opcode2[i-1]&0x10) { // BxxZAL
7597             alloc_reg(&branch_regs[i-1],i-1,31);
7598             dirty_reg(&branch_regs[i-1],31);
7599           }
7600           break;
7601       }
7602
7603       if(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)
7604       {
7605         if(rt1[i-1]==31) // JAL/JALR
7606         {
7607           // Subroutine call will return here, don't alloc any registers
7608           current.dirty=0;
7609           clear_all_regs(current.regmap);
7610           alloc_reg(&current,i,CCREG);
7611           dirty_reg(&current,CCREG);
7612         }
7613         else if(i+1<slen)
7614         {
7615           // Internal branch will jump here, match registers to caller
7616           current.dirty=0;
7617           clear_all_regs(current.regmap);
7618           alloc_reg(&current,i,CCREG);
7619           dirty_reg(&current,CCREG);
7620           for(j=i-1;j>=0;j--)
7621           {
7622             if(ba[j]==start+i*4+4) {
7623               memcpy(current.regmap,branch_regs[j].regmap,sizeof(current.regmap));
7624               current.dirty=branch_regs[j].dirty;
7625               break;
7626             }
7627           }
7628           while(j>=0) {
7629             if(ba[j]==start+i*4+4) {
7630               for(hr=0;hr<HOST_REGS;hr++) {
7631                 if(current.regmap[hr]!=branch_regs[j].regmap[hr]) {
7632                   current.regmap[hr]=-1;
7633                 }
7634                 current.dirty&=branch_regs[j].dirty;
7635               }
7636             }
7637             j--;
7638           }
7639         }
7640       }
7641     }
7642
7643     // Count cycles in between branches
7644     ccadj[i]=cc;
7645     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i]==SYSCALL||itype[i]==HLECALL))
7646     {
7647       cc=0;
7648     }
7649 #if !defined(DRC_DBG)
7650     else if(itype[i]==C2OP&&gte_cycletab[source[i]&0x3f]>2)
7651     {
7652       // GTE runs in parallel until accessed, divide by 2 for a rough guess
7653       cc+=gte_cycletab[source[i]&0x3f]/2;
7654     }
7655     else if(/*itype[i]==LOAD||itype[i]==STORE||*/itype[i]==C1LS) // load,store causes weird timing issues
7656     {
7657       cc+=2; // 2 cycle penalty (after CLOCK_DIVIDER)
7658     }
7659     else if(i>1&&itype[i]==STORE&&itype[i-1]==STORE&&itype[i-2]==STORE&&!bt[i])
7660     {
7661       cc+=4;
7662     }
7663     else if(itype[i]==C2LS)
7664     {
7665       cc+=4;
7666     }
7667 #endif
7668     else
7669     {
7670       cc++;
7671     }
7672
7673     if(!is_ds[i]) {
7674       regs[i].dirty=current.dirty;
7675       regs[i].isconst=current.isconst;
7676       memcpy(constmap[i],current_constmap,sizeof(current_constmap));
7677     }
7678     for(hr=0;hr<HOST_REGS;hr++) {
7679       if(hr!=EXCLUDE_REG&&regs[i].regmap[hr]>=0) {
7680         if(regmap_pre[i][hr]!=regs[i].regmap[hr]) {
7681           regs[i].wasconst&=~(1<<hr);
7682         }
7683       }
7684     }
7685     if(current.regmap[HOST_BTREG]==BTREG) current.regmap[HOST_BTREG]=-1;
7686     regs[i].waswritten=current.waswritten;
7687   }
7688
7689   /* Pass 4 - Cull unused host registers */
7690
7691   uint64_t nr=0;
7692
7693   for (i=slen-1;i>=0;i--)
7694   {
7695     int hr;
7696     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7697     {
7698       if(ba[i]<start || ba[i]>=(start+slen*4))
7699       {
7700         // Branch out of this block, don't need anything
7701         nr=0;
7702       }
7703       else
7704       {
7705         // Internal branch
7706         // Need whatever matches the target
7707         nr=0;
7708         int t=(ba[i]-start)>>2;
7709         for(hr=0;hr<HOST_REGS;hr++)
7710         {
7711           if(regs[i].regmap_entry[hr]>=0) {
7712             if(regs[i].regmap_entry[hr]==regs[t].regmap_entry[hr]) nr|=1<<hr;
7713           }
7714         }
7715       }
7716       // Conditional branch may need registers for following instructions
7717       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7718       {
7719         if(i<slen-2) {
7720           nr|=needed_reg[i+2];
7721           for(hr=0;hr<HOST_REGS;hr++)
7722           {
7723             if(regmap_pre[i+2][hr]>=0&&get_reg(regs[i+2].regmap_entry,regmap_pre[i+2][hr])<0) nr&=~(1<<hr);
7724             //if((regmap_entry[i+2][hr])>=0) if(!((nr>>hr)&1)) printf("%x-bogus(%d=%d)\n",start+i*4,hr,regmap_entry[i+2][hr]);
7725           }
7726         }
7727       }
7728       // Don't need stuff which is overwritten
7729       //if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
7730       //if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
7731       // Merge in delay slot
7732       for(hr=0;hr<HOST_REGS;hr++)
7733       {
7734         if(!likely[i]) {
7735           // These are overwritten unless the branch is "likely"
7736           // and the delay slot is nullified if not taken
7737           if(rt1[i+1]&&rt1[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7738           if(rt2[i+1]&&rt2[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7739         }
7740         if(us1[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7741         if(us2[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7742         if(rs1[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7743         if(rs2[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7744         if(us1[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7745         if(us2[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7746         if(rs1[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7747         if(rs2[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7748         if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) {
7749           if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
7750           if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
7751         }
7752       }
7753     }
7754     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
7755     {
7756       // SYSCALL instruction (software interrupt)
7757       nr=0;
7758     }
7759     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
7760     {
7761       // ERET instruction (return from interrupt)
7762       nr=0;
7763     }
7764     else // Non-branch
7765     {
7766       if(i<slen-1) {
7767         for(hr=0;hr<HOST_REGS;hr++) {
7768           if(regmap_pre[i+1][hr]>=0&&get_reg(regs[i+1].regmap_entry,regmap_pre[i+1][hr])<0) nr&=~(1<<hr);
7769           if(regs[i].regmap[hr]!=regmap_pre[i+1][hr]) nr&=~(1<<hr);
7770           if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
7771           if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
7772         }
7773       }
7774     }
7775     for(hr=0;hr<HOST_REGS;hr++)
7776     {
7777       // Overwritten registers are not needed
7778       if(rt1[i]&&rt1[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7779       if(rt2[i]&&rt2[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7780       if(FTEMP==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7781       // Source registers are needed
7782       if(us1[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7783       if(us2[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7784       if(rs1[i]==regmap_pre[i][hr]) nr|=1<<hr;
7785       if(rs2[i]==regmap_pre[i][hr]) nr|=1<<hr;
7786       if(us1[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7787       if(us2[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7788       if(rs1[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7789       if(rs2[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7790       if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) {
7791         if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
7792         if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
7793       }
7794       // Don't store a register immediately after writing it,
7795       // may prevent dual-issue.
7796       // But do so if this is a branch target, otherwise we
7797       // might have to load the register before the branch.
7798       if(i>0&&!bt[i]&&((regs[i].wasdirty>>hr)&1)) {
7799         if((regmap_pre[i][hr]>0&&regmap_pre[i][hr]<64&&!((unneeded_reg[i]>>regmap_pre[i][hr])&1))) {
7800           if(rt1[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7801           if(rt2[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7802         }
7803         if((regs[i].regmap_entry[hr]>0&&regs[i].regmap_entry[hr]<64&&!((unneeded_reg[i]>>regs[i].regmap_entry[hr])&1))) {
7804           if(rt1[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7805           if(rt2[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7806         }
7807       }
7808     }
7809     // Cycle count is needed at branches.  Assume it is needed at the target too.
7810     if(i==0||bt[i]||itype[i]==CJUMP||itype[i]==SPAN) {
7811       if(regmap_pre[i][HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
7812       if(regs[i].regmap_entry[HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
7813     }
7814     // Save it
7815     needed_reg[i]=nr;
7816
7817     // Deallocate unneeded registers
7818     for(hr=0;hr<HOST_REGS;hr++)
7819     {
7820       if(!((nr>>hr)&1)) {
7821         if(regs[i].regmap_entry[hr]!=CCREG) regs[i].regmap_entry[hr]=-1;
7822         if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
7823            (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7824            (regs[i].regmap[hr]&63)!=PTEMP && (regs[i].regmap[hr]&63)!=CCREG)
7825         {
7826           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7827           {
7828             if(likely[i]) {
7829               regs[i].regmap[hr]=-1;
7830               regs[i].isconst&=~(1<<hr);
7831               if(i<slen-2) {
7832                 regmap_pre[i+2][hr]=-1;
7833                 regs[i+2].wasconst&=~(1<<hr);
7834               }
7835             }
7836           }
7837         }
7838         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7839         {
7840           int d1=0,d2=0,map=0,temp=0;
7841           if(get_reg(regs[i].regmap,rt1[i+1]|64)>=0||get_reg(branch_regs[i].regmap,rt1[i+1]|64)>=0)
7842           {
7843             d1=dep1[i+1];
7844             d2=dep2[i+1];
7845           }
7846           if(itype[i+1]==STORE || itype[i+1]==STORELR ||
7847              (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
7848             map=INVCP;
7849           }
7850           if(itype[i+1]==LOADLR || itype[i+1]==STORELR ||
7851              itype[i+1]==C1LS || itype[i+1]==C2LS)
7852             temp=FTEMP;
7853           if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
7854              (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7855              (regs[i].regmap[hr]&63)!=rt1[i+1] && (regs[i].regmap[hr]&63)!=rt2[i+1] &&
7856              (regs[i].regmap[hr]^64)!=us1[i+1] && (regs[i].regmap[hr]^64)!=us2[i+1] &&
7857              (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
7858              regs[i].regmap[hr]!=rs1[i+1] && regs[i].regmap[hr]!=rs2[i+1] &&
7859              (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=PTEMP &&
7860              regs[i].regmap[hr]!=RHASH && regs[i].regmap[hr]!=RHTBL &&
7861              regs[i].regmap[hr]!=RTEMP && regs[i].regmap[hr]!=CCREG &&
7862              regs[i].regmap[hr]!=map )
7863           {
7864             regs[i].regmap[hr]=-1;
7865             regs[i].isconst&=~(1<<hr);
7866             if((branch_regs[i].regmap[hr]&63)!=rs1[i] && (branch_regs[i].regmap[hr]&63)!=rs2[i] &&
7867                (branch_regs[i].regmap[hr]&63)!=rt1[i] && (branch_regs[i].regmap[hr]&63)!=rt2[i] &&
7868                (branch_regs[i].regmap[hr]&63)!=rt1[i+1] && (branch_regs[i].regmap[hr]&63)!=rt2[i+1] &&
7869                (branch_regs[i].regmap[hr]^64)!=us1[i+1] && (branch_regs[i].regmap[hr]^64)!=us2[i+1] &&
7870                (branch_regs[i].regmap[hr]^64)!=d1 && (branch_regs[i].regmap[hr]^64)!=d2 &&
7871                branch_regs[i].regmap[hr]!=rs1[i+1] && branch_regs[i].regmap[hr]!=rs2[i+1] &&
7872                (branch_regs[i].regmap[hr]&63)!=temp && branch_regs[i].regmap[hr]!=PTEMP &&
7873                branch_regs[i].regmap[hr]!=RHASH && branch_regs[i].regmap[hr]!=RHTBL &&
7874                branch_regs[i].regmap[hr]!=RTEMP && branch_regs[i].regmap[hr]!=CCREG &&
7875                branch_regs[i].regmap[hr]!=map)
7876             {
7877               branch_regs[i].regmap[hr]=-1;
7878               branch_regs[i].regmap_entry[hr]=-1;
7879               if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7880               {
7881                 if(!likely[i]&&i<slen-2) {
7882                   regmap_pre[i+2][hr]=-1;
7883                   regs[i+2].wasconst&=~(1<<hr);
7884                 }
7885               }
7886             }
7887           }
7888         }
7889         else
7890         {
7891           // Non-branch
7892           if(i>0)
7893           {
7894             int d1=0,d2=0,map=-1,temp=-1;
7895             if(get_reg(regs[i].regmap,rt1[i]|64)>=0)
7896             {
7897               d1=dep1[i];
7898               d2=dep2[i];
7899             }
7900             if(itype[i]==STORE || itype[i]==STORELR ||
7901                       (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
7902               map=INVCP;
7903             }
7904             if(itype[i]==LOADLR || itype[i]==STORELR ||
7905                itype[i]==C1LS || itype[i]==C2LS)
7906               temp=FTEMP;
7907             if((regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7908                (regs[i].regmap[hr]^64)!=us1[i] && (regs[i].regmap[hr]^64)!=us2[i] &&
7909                (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
7910                regs[i].regmap[hr]!=rs1[i] && regs[i].regmap[hr]!=rs2[i] &&
7911                (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=map &&
7912                (itype[i]!=SPAN||regs[i].regmap[hr]!=CCREG))
7913             {
7914               if(i<slen-1&&!is_ds[i]) {
7915                 assert(regs[i].regmap[hr]<64);
7916                 if(regmap_pre[i+1][hr]!=-1 || regs[i].regmap[hr]!=-1)
7917                 if(regmap_pre[i+1][hr]!=regs[i].regmap[hr])
7918                 {
7919                   SysPrintf("fail: %x (%d %d!=%d)\n",start+i*4,hr,regmap_pre[i+1][hr],regs[i].regmap[hr]);
7920                   assert(regmap_pre[i+1][hr]==regs[i].regmap[hr]);
7921                 }
7922                 regmap_pre[i+1][hr]=-1;
7923                 if(regs[i+1].regmap_entry[hr]==CCREG) regs[i+1].regmap_entry[hr]=-1;
7924                 regs[i+1].wasconst&=~(1<<hr);
7925               }
7926               regs[i].regmap[hr]=-1;
7927               regs[i].isconst&=~(1<<hr);
7928             }
7929           }
7930         }
7931       }
7932     }
7933   }
7934
7935   /* Pass 5 - Pre-allocate registers */
7936
7937   // If a register is allocated during a loop, try to allocate it for the
7938   // entire loop, if possible.  This avoids loading/storing registers
7939   // inside of the loop.
7940
7941   signed char f_regmap[HOST_REGS];
7942   clear_all_regs(f_regmap);
7943   for(i=0;i<slen-1;i++)
7944   {
7945     if(itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7946     {
7947       if(ba[i]>=start && ba[i]<(start+i*4))
7948       if(itype[i+1]==NOP||itype[i+1]==MOV||itype[i+1]==ALU
7949       ||itype[i+1]==SHIFTIMM||itype[i+1]==IMM16||itype[i+1]==LOAD
7950       ||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS
7951       ||itype[i+1]==SHIFT||itype[i+1]==COP1
7952       ||itype[i+1]==COP2||itype[i+1]==C2LS||itype[i+1]==C2OP)
7953       {
7954         int t=(ba[i]-start)>>2;
7955         if(t>0&&(itype[t-1]!=UJUMP&&itype[t-1]!=RJUMP&&itype[t-1]!=CJUMP&&itype[t-1]!=SJUMP)) // loop_preload can't handle jumps into delay slots
7956         if(t<2||(itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||rt1[t-2]!=31) // call/ret assumes no registers allocated
7957         for(hr=0;hr<HOST_REGS;hr++)
7958         {
7959           if(regs[i].regmap[hr]>64) {
7960             if(!((regs[i].dirty>>hr)&1))
7961               f_regmap[hr]=regs[i].regmap[hr];
7962             else f_regmap[hr]=-1;
7963           }
7964           else if(regs[i].regmap[hr]>=0) {
7965             if(f_regmap[hr]!=regs[i].regmap[hr]) {
7966               // dealloc old register
7967               int n;
7968               for(n=0;n<HOST_REGS;n++)
7969               {
7970                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
7971               }
7972               // and alloc new one
7973               f_regmap[hr]=regs[i].regmap[hr];
7974             }
7975           }
7976           if(branch_regs[i].regmap[hr]>64) {
7977             if(!((branch_regs[i].dirty>>hr)&1))
7978               f_regmap[hr]=branch_regs[i].regmap[hr];
7979             else f_regmap[hr]=-1;
7980           }
7981           else if(branch_regs[i].regmap[hr]>=0) {
7982             if(f_regmap[hr]!=branch_regs[i].regmap[hr]) {
7983               // dealloc old register
7984               int n;
7985               for(n=0;n<HOST_REGS;n++)
7986               {
7987                 if(f_regmap[n]==branch_regs[i].regmap[hr]) {f_regmap[n]=-1;}
7988               }
7989               // and alloc new one
7990               f_regmap[hr]=branch_regs[i].regmap[hr];
7991             }
7992           }
7993           if(ooo[i]) {
7994             if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1])
7995               f_regmap[hr]=branch_regs[i].regmap[hr];
7996           }else{
7997             if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1])
7998               f_regmap[hr]=branch_regs[i].regmap[hr];
7999           }
8000           // Avoid dirty->clean transition
8001           #ifdef DESTRUCTIVE_WRITEBACK
8002           if(t>0) if(get_reg(regmap_pre[t],f_regmap[hr])>=0) if((regs[t].wasdirty>>get_reg(regmap_pre[t],f_regmap[hr]))&1) f_regmap[hr]=-1;
8003           #endif
8004           // This check is only strictly required in the DESTRUCTIVE_WRITEBACK
8005           // case above, however it's always a good idea.  We can't hoist the
8006           // load if the register was already allocated, so there's no point
8007           // wasting time analyzing most of these cases.  It only "succeeds"
8008           // when the mapping was different and the load can be replaced with
8009           // a mov, which is of negligible benefit.  So such cases are
8010           // skipped below.
8011           if(f_regmap[hr]>0) {
8012             if(regs[t].regmap[hr]==f_regmap[hr]||(regs[t].regmap_entry[hr]<0&&get_reg(regmap_pre[t],f_regmap[hr])<0)) {
8013               int r=f_regmap[hr];
8014               for(j=t;j<=i;j++)
8015               {
8016                 //printf("Test %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8017                 if(r<34&&((unneeded_reg[j]>>r)&1)) break;
8018                 assert(r < 64);
8019                 if(regs[j].regmap[hr]==f_regmap[hr]&&(f_regmap[hr]&63)<TEMPREG) {
8020                   //printf("Hit %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8021                   int k;
8022                   if(regs[i].regmap[hr]==-1&&branch_regs[i].regmap[hr]==-1) {
8023                     if(get_reg(regs[i+2].regmap,f_regmap[hr])>=0) break;
8024                     if(r>63) {
8025                       if(get_reg(regs[i].regmap,r&63)<0) break;
8026                       if(get_reg(branch_regs[i].regmap,r&63)<0) break;
8027                     }
8028                     k=i;
8029                     while(k>1&&regs[k-1].regmap[hr]==-1) {
8030                       if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8031                         //printf("no free regs for store %x\n",start+(k-1)*4);
8032                         break;
8033                       }
8034                       if(get_reg(regs[k-1].regmap,f_regmap[hr])>=0) {
8035                         //printf("no-match due to different register\n");
8036                         break;
8037                       }
8038                       if(itype[k-2]==UJUMP||itype[k-2]==RJUMP||itype[k-2]==CJUMP||itype[k-2]==SJUMP) {
8039                         //printf("no-match due to branch\n");
8040                         break;
8041                       }
8042                       // call/ret fast path assumes no registers allocated
8043                       if(k>2&&(itype[k-3]==UJUMP||itype[k-3]==RJUMP)&&rt1[k-3]==31) {
8044                         break;
8045                       }
8046                       assert(r < 64);
8047                       k--;
8048                     }
8049                     if(regs[k-1].regmap[hr]==f_regmap[hr]&&regmap_pre[k][hr]==f_regmap[hr]) {
8050                       //printf("Extend r%d, %x ->\n",hr,start+k*4);
8051                       while(k<i) {
8052                         regs[k].regmap_entry[hr]=f_regmap[hr];
8053                         regs[k].regmap[hr]=f_regmap[hr];
8054                         regmap_pre[k+1][hr]=f_regmap[hr];
8055                         regs[k].wasdirty&=~(1<<hr);
8056                         regs[k].dirty&=~(1<<hr);
8057                         regs[k].wasdirty|=(1<<hr)&regs[k-1].dirty;
8058                         regs[k].dirty|=(1<<hr)&regs[k].wasdirty;
8059                         regs[k].wasconst&=~(1<<hr);
8060                         regs[k].isconst&=~(1<<hr);
8061                         k++;
8062                       }
8063                     }
8064                     else {
8065                       //printf("Fail Extend r%d, %x ->\n",hr,start+k*4);
8066                       break;
8067                     }
8068                     assert(regs[i-1].regmap[hr]==f_regmap[hr]);
8069                     if(regs[i-1].regmap[hr]==f_regmap[hr]&&regmap_pre[i][hr]==f_regmap[hr]) {
8070                       //printf("OK fill %x (r%d)\n",start+i*4,hr);
8071                       regs[i].regmap_entry[hr]=f_regmap[hr];
8072                       regs[i].regmap[hr]=f_regmap[hr];
8073                       regs[i].wasdirty&=~(1<<hr);
8074                       regs[i].dirty&=~(1<<hr);
8075                       regs[i].wasdirty|=(1<<hr)&regs[i-1].dirty;
8076                       regs[i].dirty|=(1<<hr)&regs[i-1].dirty;
8077                       regs[i].wasconst&=~(1<<hr);
8078                       regs[i].isconst&=~(1<<hr);
8079                       branch_regs[i].regmap_entry[hr]=f_regmap[hr];
8080                       branch_regs[i].wasdirty&=~(1<<hr);
8081                       branch_regs[i].wasdirty|=(1<<hr)&regs[i].dirty;
8082                       branch_regs[i].regmap[hr]=f_regmap[hr];
8083                       branch_regs[i].dirty&=~(1<<hr);
8084                       branch_regs[i].dirty|=(1<<hr)&regs[i].dirty;
8085                       branch_regs[i].wasconst&=~(1<<hr);
8086                       branch_regs[i].isconst&=~(1<<hr);
8087                       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
8088                         regmap_pre[i+2][hr]=f_regmap[hr];
8089                         regs[i+2].wasdirty&=~(1<<hr);
8090                         regs[i+2].wasdirty|=(1<<hr)&regs[i].dirty;
8091                       }
8092                     }
8093                   }
8094                   for(k=t;k<j;k++) {
8095                     // Alloc register clean at beginning of loop,
8096                     // but may dirty it in pass 6
8097                     regs[k].regmap_entry[hr]=f_regmap[hr];
8098                     regs[k].regmap[hr]=f_regmap[hr];
8099                     regs[k].dirty&=~(1<<hr);
8100                     regs[k].wasconst&=~(1<<hr);
8101                     regs[k].isconst&=~(1<<hr);
8102                     if(itype[k]==UJUMP||itype[k]==RJUMP||itype[k]==CJUMP||itype[k]==SJUMP) {
8103                       branch_regs[k].regmap_entry[hr]=f_regmap[hr];
8104                       branch_regs[k].regmap[hr]=f_regmap[hr];
8105                       branch_regs[k].dirty&=~(1<<hr);
8106                       branch_regs[k].wasconst&=~(1<<hr);
8107                       branch_regs[k].isconst&=~(1<<hr);
8108                       if(itype[k]!=RJUMP&&itype[k]!=UJUMP&&(source[k]>>16)!=0x1000) {
8109                         regmap_pre[k+2][hr]=f_regmap[hr];
8110                         regs[k+2].wasdirty&=~(1<<hr);
8111                       }
8112                     }
8113                     else
8114                     {
8115                       regmap_pre[k+1][hr]=f_regmap[hr];
8116                       regs[k+1].wasdirty&=~(1<<hr);
8117                     }
8118                   }
8119                   if(regs[j].regmap[hr]==f_regmap[hr])
8120                     regs[j].regmap_entry[hr]=f_regmap[hr];
8121                   break;
8122                 }
8123                 if(j==i) break;
8124                 if(regs[j].regmap[hr]>=0)
8125                   break;
8126                 if(get_reg(regs[j].regmap,f_regmap[hr])>=0) {
8127                   //printf("no-match due to different register\n");
8128                   break;
8129                 }
8130                 if(itype[j]==UJUMP||itype[j]==RJUMP||(source[j]>>16)==0x1000)
8131                 {
8132                   // Stop on unconditional branch
8133                   break;
8134                 }
8135                 if(itype[j]==CJUMP||itype[j]==SJUMP)
8136                 {
8137                   if(ooo[j]) {
8138                     if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1])
8139                       break;
8140                   }else{
8141                     if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1])
8142                       break;
8143                   }
8144                   if(get_reg(branch_regs[j].regmap,f_regmap[hr])>=0) {
8145                     //printf("no-match due to different register (branch)\n");
8146                     break;
8147                   }
8148                 }
8149                 if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8150                   //printf("No free regs for store %x\n",start+j*4);
8151                   break;
8152                 }
8153                 assert(f_regmap[hr]<64);
8154               }
8155             }
8156           }
8157         }
8158       }
8159     }else{
8160       // Non branch or undetermined branch target
8161       for(hr=0;hr<HOST_REGS;hr++)
8162       {
8163         if(hr!=EXCLUDE_REG) {
8164           if(regs[i].regmap[hr]>64) {
8165             if(!((regs[i].dirty>>hr)&1))
8166               f_regmap[hr]=regs[i].regmap[hr];
8167           }
8168           else if(regs[i].regmap[hr]>=0) {
8169             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8170               // dealloc old register
8171               int n;
8172               for(n=0;n<HOST_REGS;n++)
8173               {
8174                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8175               }
8176               // and alloc new one
8177               f_regmap[hr]=regs[i].regmap[hr];
8178             }
8179           }
8180         }
8181       }
8182       // Try to restore cycle count at branch targets
8183       if(bt[i]) {
8184         for(j=i;j<slen-1;j++) {
8185           if(regs[j].regmap[HOST_CCREG]!=-1) break;
8186           if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8187             //printf("no free regs for store %x\n",start+j*4);
8188             break;
8189           }
8190         }
8191         if(regs[j].regmap[HOST_CCREG]==CCREG) {
8192           int k=i;
8193           //printf("Extend CC, %x -> %x\n",start+k*4,start+j*4);
8194           while(k<j) {
8195             regs[k].regmap_entry[HOST_CCREG]=CCREG;
8196             regs[k].regmap[HOST_CCREG]=CCREG;
8197             regmap_pre[k+1][HOST_CCREG]=CCREG;
8198             regs[k+1].wasdirty|=1<<HOST_CCREG;
8199             regs[k].dirty|=1<<HOST_CCREG;
8200             regs[k].wasconst&=~(1<<HOST_CCREG);
8201             regs[k].isconst&=~(1<<HOST_CCREG);
8202             k++;
8203           }
8204           regs[j].regmap_entry[HOST_CCREG]=CCREG;
8205         }
8206         // Work backwards from the branch target
8207         if(j>i&&f_regmap[HOST_CCREG]==CCREG)
8208         {
8209           //printf("Extend backwards\n");
8210           int k;
8211           k=i;
8212           while(regs[k-1].regmap[HOST_CCREG]==-1) {
8213             if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8214               //printf("no free regs for store %x\n",start+(k-1)*4);
8215               break;
8216             }
8217             k--;
8218           }
8219           if(regs[k-1].regmap[HOST_CCREG]==CCREG) {
8220             //printf("Extend CC, %x ->\n",start+k*4);
8221             while(k<=i) {
8222               regs[k].regmap_entry[HOST_CCREG]=CCREG;
8223               regs[k].regmap[HOST_CCREG]=CCREG;
8224               regmap_pre[k+1][HOST_CCREG]=CCREG;
8225               regs[k+1].wasdirty|=1<<HOST_CCREG;
8226               regs[k].dirty|=1<<HOST_CCREG;
8227               regs[k].wasconst&=~(1<<HOST_CCREG);
8228               regs[k].isconst&=~(1<<HOST_CCREG);
8229               k++;
8230             }
8231           }
8232           else {
8233             //printf("Fail Extend CC, %x ->\n",start+k*4);
8234           }
8235         }
8236       }
8237       if(itype[i]!=STORE&&itype[i]!=STORELR&&itype[i]!=C1LS&&itype[i]!=SHIFT&&
8238          itype[i]!=NOP&&itype[i]!=MOV&&itype[i]!=ALU&&itype[i]!=SHIFTIMM&&
8239          itype[i]!=IMM16&&itype[i]!=LOAD&&itype[i]!=COP1)
8240       {
8241         memcpy(f_regmap,regs[i].regmap,sizeof(f_regmap));
8242       }
8243     }
8244   }
8245
8246   // This allocates registers (if possible) one instruction prior
8247   // to use, which can avoid a load-use penalty on certain CPUs.
8248   for(i=0;i<slen-1;i++)
8249   {
8250     if(!i||(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP))
8251     {
8252       if(!bt[i+1])
8253       {
8254         if(itype[i]==ALU||itype[i]==MOV||itype[i]==LOAD||itype[i]==SHIFTIMM||itype[i]==IMM16
8255            ||((itype[i]==COP1||itype[i]==COP2)&&opcode2[i]<3))
8256         {
8257           if(rs1[i+1]) {
8258             if((hr=get_reg(regs[i+1].regmap,rs1[i+1]))>=0)
8259             {
8260               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8261               {
8262                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8263                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8264                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8265                 regs[i].isconst&=~(1<<hr);
8266                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8267                 constmap[i][hr]=constmap[i+1][hr];
8268                 regs[i+1].wasdirty&=~(1<<hr);
8269                 regs[i].dirty&=~(1<<hr);
8270               }
8271             }
8272           }
8273           if(rs2[i+1]) {
8274             if((hr=get_reg(regs[i+1].regmap,rs2[i+1]))>=0)
8275             {
8276               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8277               {
8278                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8279                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8280                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8281                 regs[i].isconst&=~(1<<hr);
8282                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8283                 constmap[i][hr]=constmap[i+1][hr];
8284                 regs[i+1].wasdirty&=~(1<<hr);
8285                 regs[i].dirty&=~(1<<hr);
8286               }
8287             }
8288           }
8289           // Preload target address for load instruction (non-constant)
8290           if(itype[i+1]==LOAD&&rs1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8291             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8292             {
8293               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8294               {
8295                 regs[i].regmap[hr]=rs1[i+1];
8296                 regmap_pre[i+1][hr]=rs1[i+1];
8297                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8298                 regs[i].isconst&=~(1<<hr);
8299                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8300                 constmap[i][hr]=constmap[i+1][hr];
8301                 regs[i+1].wasdirty&=~(1<<hr);
8302                 regs[i].dirty&=~(1<<hr);
8303               }
8304             }
8305           }
8306           // Load source into target register
8307           if(lt1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8308             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8309             {
8310               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8311               {
8312                 regs[i].regmap[hr]=rs1[i+1];
8313                 regmap_pre[i+1][hr]=rs1[i+1];
8314                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8315                 regs[i].isconst&=~(1<<hr);
8316                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8317                 constmap[i][hr]=constmap[i+1][hr];
8318                 regs[i+1].wasdirty&=~(1<<hr);
8319                 regs[i].dirty&=~(1<<hr);
8320               }
8321             }
8322           }
8323           // Address for store instruction (non-constant)
8324           if(itype[i+1]==STORE||itype[i+1]==STORELR
8325              ||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SB/SH/SW/SD/SWC1/SDC1/SWC2/SDC2
8326             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8327               hr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1);
8328               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8329               else {regs[i+1].regmap[hr]=AGEN1+((i+1)&1);regs[i+1].isconst&=~(1<<hr);}
8330               assert(hr>=0);
8331               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8332               {
8333                 regs[i].regmap[hr]=rs1[i+1];
8334                 regmap_pre[i+1][hr]=rs1[i+1];
8335                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8336                 regs[i].isconst&=~(1<<hr);
8337                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8338                 constmap[i][hr]=constmap[i+1][hr];
8339                 regs[i+1].wasdirty&=~(1<<hr);
8340                 regs[i].dirty&=~(1<<hr);
8341               }
8342             }
8343           }
8344           if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) { // LWC1/LDC1, LWC2/LDC2
8345             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8346               int nr;
8347               hr=get_reg(regs[i+1].regmap,FTEMP);
8348               assert(hr>=0);
8349               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8350               {
8351                 regs[i].regmap[hr]=rs1[i+1];
8352                 regmap_pre[i+1][hr]=rs1[i+1];
8353                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8354                 regs[i].isconst&=~(1<<hr);
8355                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8356                 constmap[i][hr]=constmap[i+1][hr];
8357                 regs[i+1].wasdirty&=~(1<<hr);
8358                 regs[i].dirty&=~(1<<hr);
8359               }
8360               else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
8361               {
8362                 // move it to another register
8363                 regs[i+1].regmap[hr]=-1;
8364                 regmap_pre[i+2][hr]=-1;
8365                 regs[i+1].regmap[nr]=FTEMP;
8366                 regmap_pre[i+2][nr]=FTEMP;
8367                 regs[i].regmap[nr]=rs1[i+1];
8368                 regmap_pre[i+1][nr]=rs1[i+1];
8369                 regs[i+1].regmap_entry[nr]=rs1[i+1];
8370                 regs[i].isconst&=~(1<<nr);
8371                 regs[i+1].isconst&=~(1<<nr);
8372                 regs[i].dirty&=~(1<<nr);
8373                 regs[i+1].wasdirty&=~(1<<nr);
8374                 regs[i+1].dirty&=~(1<<nr);
8375                 regs[i+2].wasdirty&=~(1<<nr);
8376               }
8377             }
8378           }
8379           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR/*||itype[i+1]==C1LS||||itype[i+1]==C2LS*/) {
8380             if(itype[i+1]==LOAD)
8381               hr=get_reg(regs[i+1].regmap,rt1[i+1]);
8382             if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) // LWC1/LDC1, LWC2/LDC2
8383               hr=get_reg(regs[i+1].regmap,FTEMP);
8384             if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1/SWC2/SDC2
8385               hr=get_reg(regs[i+1].regmap,AGEN1+((i+1)&1));
8386               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8387             }
8388             if(hr>=0&&regs[i].regmap[hr]<0) {
8389               int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
8390               if(rs>=0&&((regs[i+1].wasconst>>rs)&1)) {
8391                 regs[i].regmap[hr]=AGEN1+((i+1)&1);
8392                 regmap_pre[i+1][hr]=AGEN1+((i+1)&1);
8393                 regs[i+1].regmap_entry[hr]=AGEN1+((i+1)&1);
8394                 regs[i].isconst&=~(1<<hr);
8395                 regs[i+1].wasdirty&=~(1<<hr);
8396                 regs[i].dirty&=~(1<<hr);
8397               }
8398             }
8399           }
8400         }
8401       }
8402     }
8403   }
8404
8405   /* Pass 6 - Optimize clean/dirty state */
8406   clean_registers(0,slen-1,1);
8407
8408   /* Pass 7 - Identify 32-bit registers */
8409   for (i=slen-1;i>=0;i--)
8410   {
8411     if(itype[i]==CJUMP||itype[i]==SJUMP)
8412     {
8413       // Conditional branch
8414       if((source[i]>>16)!=0x1000&&i<slen-2) {
8415         // Mark this address as a branch target since it may be called
8416         // upon return from interrupt
8417         bt[i+2]=1;
8418       }
8419     }
8420   }
8421
8422   if(itype[slen-1]==SPAN) {
8423     bt[slen-1]=1; // Mark as a branch target so instruction can restart after exception
8424   }
8425
8426 #ifdef DISASM
8427   /* Debug/disassembly */
8428   for(i=0;i<slen;i++)
8429   {
8430     printf("U:");
8431     int r;
8432     for(r=1;r<=CCREG;r++) {
8433       if((unneeded_reg[i]>>r)&1) {
8434         if(r==HIREG) printf(" HI");
8435         else if(r==LOREG) printf(" LO");
8436         else printf(" r%d",r);
8437       }
8438     }
8439     printf("\n");
8440     #if defined(__i386__) || defined(__x86_64__)
8441     printf("pre: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7]);
8442     #endif
8443     #ifdef __arm__
8444     printf("pre: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][4],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7],regmap_pre[i][8],regmap_pre[i][9],regmap_pre[i][10],regmap_pre[i][12]);
8445     #endif
8446     printf("needs: ");
8447     if(needed_reg[i]&1) printf("eax ");
8448     if((needed_reg[i]>>1)&1) printf("ecx ");
8449     if((needed_reg[i]>>2)&1) printf("edx ");
8450     if((needed_reg[i]>>3)&1) printf("ebx ");
8451     if((needed_reg[i]>>5)&1) printf("ebp ");
8452     if((needed_reg[i]>>6)&1) printf("esi ");
8453     if((needed_reg[i]>>7)&1) printf("edi ");
8454     printf("\n");
8455     #if defined(__i386__) || defined(__x86_64__)
8456     printf("entry: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7]);
8457     printf("dirty: ");
8458     if(regs[i].wasdirty&1) printf("eax ");
8459     if((regs[i].wasdirty>>1)&1) printf("ecx ");
8460     if((regs[i].wasdirty>>2)&1) printf("edx ");
8461     if((regs[i].wasdirty>>3)&1) printf("ebx ");
8462     if((regs[i].wasdirty>>5)&1) printf("ebp ");
8463     if((regs[i].wasdirty>>6)&1) printf("esi ");
8464     if((regs[i].wasdirty>>7)&1) printf("edi ");
8465     #endif
8466     #ifdef __arm__
8467     printf("entry: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[4],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7],regs[i].regmap_entry[8],regs[i].regmap_entry[9],regs[i].regmap_entry[10],regs[i].regmap_entry[12]);
8468     printf("dirty: ");
8469     if(regs[i].wasdirty&1) printf("r0 ");
8470     if((regs[i].wasdirty>>1)&1) printf("r1 ");
8471     if((regs[i].wasdirty>>2)&1) printf("r2 ");
8472     if((regs[i].wasdirty>>3)&1) printf("r3 ");
8473     if((regs[i].wasdirty>>4)&1) printf("r4 ");
8474     if((regs[i].wasdirty>>5)&1) printf("r5 ");
8475     if((regs[i].wasdirty>>6)&1) printf("r6 ");
8476     if((regs[i].wasdirty>>7)&1) printf("r7 ");
8477     if((regs[i].wasdirty>>8)&1) printf("r8 ");
8478     if((regs[i].wasdirty>>9)&1) printf("r9 ");
8479     if((regs[i].wasdirty>>10)&1) printf("r10 ");
8480     if((regs[i].wasdirty>>12)&1) printf("r12 ");
8481     #endif
8482     printf("\n");
8483     disassemble_inst(i);
8484     //printf ("ccadj[%d] = %d\n",i,ccadj[i]);
8485     #if defined(__i386__) || defined(__x86_64__)
8486     printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7]);
8487     if(regs[i].dirty&1) printf("eax ");
8488     if((regs[i].dirty>>1)&1) printf("ecx ");
8489     if((regs[i].dirty>>2)&1) printf("edx ");
8490     if((regs[i].dirty>>3)&1) printf("ebx ");
8491     if((regs[i].dirty>>5)&1) printf("ebp ");
8492     if((regs[i].dirty>>6)&1) printf("esi ");
8493     if((regs[i].dirty>>7)&1) printf("edi ");
8494     #endif
8495     #ifdef __arm__
8496     printf("r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[4],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7],regs[i].regmap[8],regs[i].regmap[9],regs[i].regmap[10],regs[i].regmap[12]);
8497     if(regs[i].dirty&1) printf("r0 ");
8498     if((regs[i].dirty>>1)&1) printf("r1 ");
8499     if((regs[i].dirty>>2)&1) printf("r2 ");
8500     if((regs[i].dirty>>3)&1) printf("r3 ");
8501     if((regs[i].dirty>>4)&1) printf("r4 ");
8502     if((regs[i].dirty>>5)&1) printf("r5 ");
8503     if((regs[i].dirty>>6)&1) printf("r6 ");
8504     if((regs[i].dirty>>7)&1) printf("r7 ");
8505     if((regs[i].dirty>>8)&1) printf("r8 ");
8506     if((regs[i].dirty>>9)&1) printf("r9 ");
8507     if((regs[i].dirty>>10)&1) printf("r10 ");
8508     if((regs[i].dirty>>12)&1) printf("r12 ");
8509     #endif
8510     printf("\n");
8511     if(regs[i].isconst) {
8512       printf("constants: ");
8513       #if defined(__i386__) || defined(__x86_64__)
8514       if(regs[i].isconst&1) printf("eax=%x ",(u_int)constmap[i][0]);
8515       if((regs[i].isconst>>1)&1) printf("ecx=%x ",(u_int)constmap[i][1]);
8516       if((regs[i].isconst>>2)&1) printf("edx=%x ",(u_int)constmap[i][2]);
8517       if((regs[i].isconst>>3)&1) printf("ebx=%x ",(u_int)constmap[i][3]);
8518       if((regs[i].isconst>>5)&1) printf("ebp=%x ",(u_int)constmap[i][5]);
8519       if((regs[i].isconst>>6)&1) printf("esi=%x ",(u_int)constmap[i][6]);
8520       if((regs[i].isconst>>7)&1) printf("edi=%x ",(u_int)constmap[i][7]);
8521       #endif
8522       #ifdef __arm__
8523       int r;
8524       for (r = 0; r < ARRAY_SIZE(constmap[i]); r++)
8525         if ((regs[i].isconst >> r) & 1)
8526           printf(" r%d=%x", r, (u_int)constmap[i][r]);
8527       #endif
8528       printf("\n");
8529     }
8530     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
8531       #if defined(__i386__) || defined(__x86_64__)
8532       printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
8533       if(branch_regs[i].dirty&1) printf("eax ");
8534       if((branch_regs[i].dirty>>1)&1) printf("ecx ");
8535       if((branch_regs[i].dirty>>2)&1) printf("edx ");
8536       if((branch_regs[i].dirty>>3)&1) printf("ebx ");
8537       if((branch_regs[i].dirty>>5)&1) printf("ebp ");
8538       if((branch_regs[i].dirty>>6)&1) printf("esi ");
8539       if((branch_regs[i].dirty>>7)&1) printf("edi ");
8540       #endif
8541       #ifdef __arm__
8542       printf("branch(%d): r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[4],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7],branch_regs[i].regmap[8],branch_regs[i].regmap[9],branch_regs[i].regmap[10],branch_regs[i].regmap[12]);
8543       if(branch_regs[i].dirty&1) printf("r0 ");
8544       if((branch_regs[i].dirty>>1)&1) printf("r1 ");
8545       if((branch_regs[i].dirty>>2)&1) printf("r2 ");
8546       if((branch_regs[i].dirty>>3)&1) printf("r3 ");
8547       if((branch_regs[i].dirty>>4)&1) printf("r4 ");
8548       if((branch_regs[i].dirty>>5)&1) printf("r5 ");
8549       if((branch_regs[i].dirty>>6)&1) printf("r6 ");
8550       if((branch_regs[i].dirty>>7)&1) printf("r7 ");
8551       if((branch_regs[i].dirty>>8)&1) printf("r8 ");
8552       if((branch_regs[i].dirty>>9)&1) printf("r9 ");
8553       if((branch_regs[i].dirty>>10)&1) printf("r10 ");
8554       if((branch_regs[i].dirty>>12)&1) printf("r12 ");
8555       #endif
8556     }
8557   }
8558 #endif // DISASM
8559
8560   /* Pass 8 - Assembly */
8561   linkcount=0;stubcount=0;
8562   ds=0;is_delayslot=0;
8563   u_int dirty_pre=0;
8564   void *beginning=start_block();
8565   if((u_int)addr&1) {
8566     ds=1;
8567     pagespan_ds();
8568   }
8569   void *instr_addr0_override = NULL;
8570
8571   if (start == 0x80030000) {
8572     // nasty hack for fastbios thing
8573     // override block entry to this code
8574     instr_addr0_override = out;
8575     emit_movimm(start,0);
8576     // abuse io address var as a flag that we
8577     // have already returned here once
8578     emit_readword(&address,1);
8579     emit_writeword(0,&pcaddr);
8580     emit_writeword(0,&address);
8581     emit_cmp(0,1);
8582     emit_jne(new_dyna_leave);
8583   }
8584   for(i=0;i<slen;i++)
8585   {
8586     //if(ds) printf("ds: ");
8587     disassemble_inst(i);
8588     if(ds) {
8589       ds=0; // Skip delay slot
8590       if(bt[i]) assem_debug("OOPS - branch into delay slot\n");
8591       instr_addr[i] = NULL;
8592     } else {
8593       speculate_register_values(i);
8594       #ifndef DESTRUCTIVE_WRITEBACK
8595       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8596       {
8597         wb_valid(regmap_pre[i],regs[i].regmap_entry,dirty_pre,regs[i].wasdirty,unneeded_reg[i]);
8598       }
8599       if((itype[i]==CJUMP||itype[i]==SJUMP)&&!likely[i]) {
8600         dirty_pre=branch_regs[i].dirty;
8601       }else{
8602         dirty_pre=regs[i].dirty;
8603       }
8604       #endif
8605       // write back
8606       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8607       {
8608         wb_invalidate(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,unneeded_reg[i]);
8609         loop_preload(regmap_pre[i],regs[i].regmap_entry);
8610       }
8611       // branch target entry point
8612       instr_addr[i] = out;
8613       assem_debug("<->\n");
8614       drc_dbg_emit_do_cmp(i);
8615
8616       // load regs
8617       if(regs[i].regmap_entry[HOST_CCREG]==CCREG&&regs[i].regmap[HOST_CCREG]!=CCREG)
8618         wb_register(CCREG,regs[i].regmap_entry,regs[i].wasdirty);
8619       load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i],rs2[i]);
8620       address_generation(i,&regs[i],regs[i].regmap_entry);
8621       load_consts(regmap_pre[i],regs[i].regmap,i);
8622       if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8623       {
8624         // Load the delay slot registers if necessary
8625         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i]&&(rs1[i+1]!=rt1[i]||rt1[i]==0))
8626           load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8627         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i]&&(rs2[i+1]!=rt1[i]||rt1[i]==0))
8628           load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8629         if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a)
8630           load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8631       }
8632       else if(i+1<slen)
8633       {
8634         // Preload registers for following instruction
8635         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
8636           if(rs1[i+1]!=rt1[i]&&rs1[i+1]!=rt2[i])
8637             load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8638         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
8639           if(rs2[i+1]!=rt1[i]&&rs2[i+1]!=rt2[i])
8640             load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8641       }
8642       // TODO: if(is_ooo(i)) address_generation(i+1);
8643       if(itype[i]==CJUMP)
8644         load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
8645       if(itype[i]==STORE||itype[i]==STORELR||(opcode[i]&0x3b)==0x39||(opcode[i]&0x3b)==0x3a)
8646         load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8647       // assemble
8648       switch(itype[i]) {
8649         case ALU:
8650           alu_assemble(i,&regs[i]);break;
8651         case IMM16:
8652           imm16_assemble(i,&regs[i]);break;
8653         case SHIFT:
8654           shift_assemble(i,&regs[i]);break;
8655         case SHIFTIMM:
8656           shiftimm_assemble(i,&regs[i]);break;
8657         case LOAD:
8658           load_assemble(i,&regs[i]);break;
8659         case LOADLR:
8660           loadlr_assemble(i,&regs[i]);break;
8661         case STORE:
8662           store_assemble(i,&regs[i]);break;
8663         case STORELR:
8664           storelr_assemble(i,&regs[i]);break;
8665         case COP0:
8666           cop0_assemble(i,&regs[i]);break;
8667         case COP1:
8668           cop1_assemble(i,&regs[i]);break;
8669         case C1LS:
8670           c1ls_assemble(i,&regs[i]);break;
8671         case COP2:
8672           cop2_assemble(i,&regs[i]);break;
8673         case C2LS:
8674           c2ls_assemble(i,&regs[i]);break;
8675         case C2OP:
8676           c2op_assemble(i,&regs[i]);break;
8677         case MULTDIV:
8678           multdiv_assemble(i,&regs[i]);break;
8679         case MOV:
8680           mov_assemble(i,&regs[i]);break;
8681         case SYSCALL:
8682           syscall_assemble(i,&regs[i]);break;
8683         case HLECALL:
8684           hlecall_assemble(i,&regs[i]);break;
8685         case INTCALL:
8686           intcall_assemble(i,&regs[i]);break;
8687         case UJUMP:
8688           ujump_assemble(i,&regs[i]);ds=1;break;
8689         case RJUMP:
8690           rjump_assemble(i,&regs[i]);ds=1;break;
8691         case CJUMP:
8692           cjump_assemble(i,&regs[i]);ds=1;break;
8693         case SJUMP:
8694           sjump_assemble(i,&regs[i]);ds=1;break;
8695         case SPAN:
8696           pagespan_assemble(i,&regs[i]);break;
8697       }
8698       if(itype[i]==UJUMP||itype[i]==RJUMP||(source[i]>>16)==0x1000)
8699         literal_pool(1024);
8700       else
8701         literal_pool_jumpover(256);
8702     }
8703   }
8704   //assert(itype[i-2]==UJUMP||itype[i-2]==RJUMP||(source[i-2]>>16)==0x1000);
8705   // If the block did not end with an unconditional branch,
8706   // add a jump to the next instruction.
8707   if(i>1) {
8708     if(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000&&itype[i-1]!=SPAN) {
8709       assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8710       assert(i==slen);
8711       if(itype[i-2]!=CJUMP&&itype[i-2]!=SJUMP) {
8712         store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8713         if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8714           emit_loadreg(CCREG,HOST_CCREG);
8715         emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8716       }
8717       else if(!likely[i-2])
8718       {
8719         store_regs_bt(branch_regs[i-2].regmap,branch_regs[i-2].dirty,start+i*4);
8720         assert(branch_regs[i-2].regmap[HOST_CCREG]==CCREG);
8721       }
8722       else
8723       {
8724         store_regs_bt(regs[i-2].regmap,regs[i-2].dirty,start+i*4);
8725         assert(regs[i-2].regmap[HOST_CCREG]==CCREG);
8726       }
8727       add_to_linker(out,start+i*4,0);
8728       emit_jmp(0);
8729     }
8730   }
8731   else
8732   {
8733     assert(i>0);
8734     assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8735     store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8736     if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8737       emit_loadreg(CCREG,HOST_CCREG);
8738     emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8739     add_to_linker(out,start+i*4,0);
8740     emit_jmp(0);
8741   }
8742
8743   // TODO: delay slot stubs?
8744   // Stubs
8745   for(i=0;i<stubcount;i++)
8746   {
8747     switch(stubs[i].type)
8748     {
8749       case LOADB_STUB:
8750       case LOADH_STUB:
8751       case LOADW_STUB:
8752       case LOADD_STUB:
8753       case LOADBU_STUB:
8754       case LOADHU_STUB:
8755         do_readstub(i);break;
8756       case STOREB_STUB:
8757       case STOREH_STUB:
8758       case STOREW_STUB:
8759       case STORED_STUB:
8760         do_writestub(i);break;
8761       case CC_STUB:
8762         do_ccstub(i);break;
8763       case INVCODE_STUB:
8764         do_invstub(i);break;
8765       case FP_STUB:
8766         do_cop1stub(i);break;
8767       case STORELR_STUB:
8768         do_unalignedwritestub(i);break;
8769     }
8770   }
8771
8772   if (instr_addr0_override)
8773     instr_addr[0] = instr_addr0_override;
8774
8775   /* Pass 9 - Linker */
8776   for(i=0;i<linkcount;i++)
8777   {
8778     assem_debug("%p -> %8x\n",link_addr[i].addr,link_addr[i].target);
8779     literal_pool(64);
8780     if (!link_addr[i].ext)
8781     {
8782       void *stub = out;
8783       void *addr = check_addr(link_addr[i].target);
8784       emit_extjump(link_addr[i].addr, link_addr[i].target);
8785       if (addr) {
8786         set_jump_target(link_addr[i].addr, addr);
8787         add_link(link_addr[i].target,stub);
8788       }
8789       else
8790         set_jump_target(link_addr[i].addr, stub);
8791     }
8792     else
8793     {
8794       // Internal branch
8795       int target=(link_addr[i].target-start)>>2;
8796       assert(target>=0&&target<slen);
8797       assert(instr_addr[target]);
8798       //#ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
8799       //set_jump_target_fillslot(link_addr[i].addr,instr_addr[target],link_addr[i].ext>>1);
8800       //#else
8801       set_jump_target(link_addr[i].addr, instr_addr[target]);
8802       //#endif
8803     }
8804   }
8805   // External Branch Targets (jump_in)
8806   if(copy+slen*4>(void *)shadow+sizeof(shadow)) copy=shadow;
8807   for(i=0;i<slen;i++)
8808   {
8809     if(bt[i]||i==0)
8810     {
8811       if(instr_addr[i]) // TODO - delay slots (=null)
8812       {
8813         u_int vaddr=start+i*4;
8814         u_int page=get_page(vaddr);
8815         u_int vpage=get_vpage(vaddr);
8816         literal_pool(256);
8817         {
8818           assem_debug("%p (%d) <- %8x\n",instr_addr[i],i,start+i*4);
8819           assem_debug("jump_in: %x\n",start+i*4);
8820           ll_add(jump_dirty+vpage,vaddr,out);
8821           void *entry_point = do_dirty_stub(i);
8822           ll_add_flags(jump_in+page,vaddr,state_rflags,entry_point);
8823           // If there was an existing entry in the hash table,
8824           // replace it with the new address.
8825           // Don't add new entries.  We'll insert the
8826           // ones that actually get used in check_addr().
8827           struct ht_entry *ht_bin = hash_table_get(vaddr);
8828           if (ht_bin->vaddr[0] == vaddr)
8829             ht_bin->tcaddr[0] = entry_point;
8830           if (ht_bin->vaddr[1] == vaddr)
8831             ht_bin->tcaddr[1] = entry_point;
8832         }
8833       }
8834     }
8835   }
8836   // Write out the literal pool if necessary
8837   literal_pool(0);
8838   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
8839   // Align code
8840   if(((u_int)out)&7) emit_addnop(13);
8841   #endif
8842   assert(out - (u_char *)beginning < MAX_OUTPUT_BLOCK_SIZE);
8843   //printf("shadow buffer: %p-%p\n",copy,(u_char *)copy+slen*4);
8844   memcpy(copy,source,slen*4);
8845   copy+=slen*4;
8846
8847   end_block(beginning);
8848
8849   // If we're within 256K of the end of the buffer,
8850   // start over from the beginning. (Is 256K enough?)
8851   if (out > translation_cache+(1<<TARGET_SIZE_2)-MAX_OUTPUT_BLOCK_SIZE)
8852     out = translation_cache;
8853
8854   // Trap writes to any of the pages we compiled
8855   for(i=start>>12;i<=(start+slen*4)>>12;i++) {
8856     invalid_code[i]=0;
8857   }
8858   inv_code_start=inv_code_end=~0;
8859
8860   // for PCSX we need to mark all mirrors too
8861   if(get_page(start)<(RAM_SIZE>>12))
8862     for(i=start>>12;i<=(start+slen*4)>>12;i++)
8863       invalid_code[((u_int)0x00000000>>12)|(i&0x1ff)]=
8864       invalid_code[((u_int)0x80000000>>12)|(i&0x1ff)]=
8865       invalid_code[((u_int)0xa0000000>>12)|(i&0x1ff)]=0;
8866
8867   /* Pass 10 - Free memory by expiring oldest blocks */
8868
8869   int end=(((out-translation_cache)>>(TARGET_SIZE_2-16))+16384)&65535;
8870   while(expirep!=end)
8871   {
8872     int shift=TARGET_SIZE_2-3; // Divide into 8 blocks
8873     uintptr_t base=(uintptr_t)translation_cache+((expirep>>13)<<shift); // Base address of this block
8874     inv_debug("EXP: Phase %d\n",expirep);
8875     switch((expirep>>11)&3)
8876     {
8877       case 0:
8878         // Clear jump_in and jump_dirty
8879         ll_remove_matching_addrs(jump_in+(expirep&2047),base,shift);
8880         ll_remove_matching_addrs(jump_dirty+(expirep&2047),base,shift);
8881         ll_remove_matching_addrs(jump_in+2048+(expirep&2047),base,shift);
8882         ll_remove_matching_addrs(jump_dirty+2048+(expirep&2047),base,shift);
8883         break;
8884       case 1:
8885         // Clear pointers
8886         ll_kill_pointers(jump_out[expirep&2047],base,shift);
8887         ll_kill_pointers(jump_out[(expirep&2047)+2048],base,shift);
8888         break;
8889       case 2:
8890         // Clear hash table
8891         for(i=0;i<32;i++) {
8892           struct ht_entry *ht_bin = &hash_table[((expirep&2047)<<5)+i];
8893           if (((uintptr_t)ht_bin->tcaddr[1]>>shift) == (base>>shift) ||
8894              (((uintptr_t)ht_bin->tcaddr[1]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
8895             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[1],ht_bin->tcaddr[1]);
8896             ht_bin->vaddr[1] = -1;
8897             ht_bin->tcaddr[1] = NULL;
8898           }
8899           if (((uintptr_t)ht_bin->tcaddr[0]>>shift) == (base>>shift) ||
8900              (((uintptr_t)ht_bin->tcaddr[0]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
8901             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[0],ht_bin->tcaddr[0]);
8902             ht_bin->vaddr[0] = ht_bin->vaddr[1];
8903             ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
8904             ht_bin->vaddr[1] = -1;
8905             ht_bin->tcaddr[1] = NULL;
8906           }
8907         }
8908         break;
8909       case 3:
8910         // Clear jump_out
8911         #if defined(__arm__) || defined(__aarch64__)
8912         if((expirep&2047)==0)
8913           do_clear_cache();
8914         #endif
8915         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
8916         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
8917         break;
8918     }
8919     expirep=(expirep+1)&65535;
8920   }
8921   return 0;
8922 }
8923
8924 // vim:shiftwidth=2:expandtab