libpcsxcore/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2011 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdlib.h>
  22 #include <stdint.h> //include for uint64_t
  23 #include <assert.h>
  24 #include <errno.h>
  25 #include <sys/mman.h>
  26 #ifdef __MACH__
  27 #include <libkern/OSCacheControl.h>
  28 #endif
  29 #ifdef _3DS
  30 #include <3ds_utils.h>
  31 #endif
  32 #ifdef VITA
  33 #include <psp2/kernel/sysmem.h>
  34 static int sceBlock;
  35 #endif
  36
  37 #include "new_dynarec_config.h"
  38 #include "../psxhle.h" //emulator interface
  39 #include "emu_if.h" //emulator interface
  40
  41 #ifndef ARRAY_SIZE
  42 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
  43 #endif
  44
  45 //#define DISASM
  46 //#define assem_debug printf
  47 //#define inv_debug printf
  48 #define assem_debug(...)
  49 #define inv_debug(...)
  50
  51 #ifdef __i386__
  52 #include "assem_x86.h"
  53 #endif
  54 #ifdef __x86_64__
  55 #include "assem_x64.h"
  56 #endif
  57 #ifdef __arm__
  58 #include "assem_arm.h"
  59 #endif
  60
  61 #define MAXBLOCK 4096
  62 #define MAX_OUTPUT_BLOCK_SIZE 262144
  63
  64 // stubs
  65 enum stub_type {
  66   CC_STUB = 1,
  67   FP_STUB = 2,
  68   LOADB_STUB = 3,
  69   LOADH_STUB = 4,
  70   LOADW_STUB = 5,
  71   LOADD_STUB = 6,
  72   LOADBU_STUB = 7,
  73   LOADHU_STUB = 8,
  74   STOREB_STUB = 9,
  75   STOREH_STUB = 10,
  76   STOREW_STUB = 11,
  77   STORED_STUB = 12,
  78   STORELR_STUB = 13,
  79   INVCODE_STUB = 14,
  80 };
  81
  82 struct regstat
  83 {
  84   signed char regmap_entry[HOST_REGS];
  85   signed char regmap[HOST_REGS];
  86   uint64_t wasdirty;
  87   uint64_t dirty;
  88   uint64_t u;
  89   u_int wasconst;
  90   u_int isconst;
  91   u_int loadedconst;             // host regs that have constants loaded
  92   u_int waswritten;              // MIPS regs that were used as store base before
  93 };
  94
  95 // note: asm depends on this layout
  96 struct ll_entry
  97 {
  98   u_int vaddr;
  99   u_int reg_sv_flags;
 100   void *addr;
 101   struct ll_entry *next;
 102 };
 103
 104 struct ht_entry
 105 {
 106   u_int vaddr[2];
 107   void *tcaddr[2];
 108 };
 109
 110 struct code_stub
 111 {
 112   enum stub_type type;
 113   void *addr;
 114   void *retaddr;
 115   u_int a;
 116   uintptr_t b;
 117   uintptr_t c;
 118   u_int d;
 119   u_int e;
 120 };
 121
 122 struct link_entry
 123 {
 124   void *addr;
 125   u_int target;
 126   u_int ext;
 127 };
 128
 129   // used by asm:
 130   u_char *out;
 131   struct ht_entry hash_table[65536]  __attribute__((aligned(16)));
 132   struct ll_entry *jump_in[4096] __attribute__((aligned(16)));
 133   struct ll_entry *jump_dirty[4096];
 134
 135   static struct ll_entry *jump_out[4096];
 136   static u_int start;
 137   static u_int *source;
 138   static char insn[MAXBLOCK][10];
 139   static u_char itype[MAXBLOCK];
 140   static u_char opcode[MAXBLOCK];
 141   static u_char opcode2[MAXBLOCK];
 142   static u_char bt[MAXBLOCK];
 143   static u_char rs1[MAXBLOCK];
 144   static u_char rs2[MAXBLOCK];
 145   static u_char rt1[MAXBLOCK];
 146   static u_char rt2[MAXBLOCK];
 147   static u_char us1[MAXBLOCK];
 148   static u_char us2[MAXBLOCK];
 149   static u_char dep1[MAXBLOCK];
 150   static u_char dep2[MAXBLOCK];
 151   static u_char lt1[MAXBLOCK];
 152   static uint64_t gte_rs[MAXBLOCK]; // gte: 32 data and 32 ctl regs
 153   static uint64_t gte_rt[MAXBLOCK];
 154   static uint64_t gte_unneeded[MAXBLOCK];
 155   static u_int smrv[32]; // speculated MIPS register values
 156   static u_int smrv_strong; // mask or regs that are likely to have correct values
 157   static u_int smrv_weak; // same, but somewhat less likely
 158   static u_int smrv_strong_next; // same, but after current insn executes
 159   static u_int smrv_weak_next;
 160   static int imm[MAXBLOCK];
 161   static u_int ba[MAXBLOCK];
 162   static char likely[MAXBLOCK];
 163   static char is_ds[MAXBLOCK];
 164   static char ooo[MAXBLOCK];
 165   static uint64_t unneeded_reg[MAXBLOCK];
 166   static uint64_t branch_unneeded_reg[MAXBLOCK];
 167   static signed char regmap_pre[MAXBLOCK][HOST_REGS];
 168   static uint64_t current_constmap[HOST_REGS];
 169   static uint64_t constmap[MAXBLOCK][HOST_REGS];
 170   static struct regstat regs[MAXBLOCK];
 171   static struct regstat branch_regs[MAXBLOCK];
 172   static signed char minimum_free_regs[MAXBLOCK];
 173   static u_int needed_reg[MAXBLOCK];
 174   static u_int wont_dirty[MAXBLOCK];
 175   static u_int will_dirty[MAXBLOCK];
 176   static int ccadj[MAXBLOCK];
 177   static int slen;
 178   static void *instr_addr[MAXBLOCK];
 179   static struct link_entry link_addr[MAXBLOCK];
 180   static int linkcount;
 181   static struct code_stub stubs[MAXBLOCK*3];
 182   static int stubcount;
 183   static u_int literals[1024][2];
 184   static int literalcount;
 185   static int is_delayslot;
 186   static char shadow[1048576]  __attribute__((aligned(16)));
 187   static void *copy;
 188   static int expirep;
 189   static u_int stop_after_jal;
 190 #ifndef RAM_FIXED
 191   static uintptr_t ram_offset;
 192 #else
 193   static const uintptr_t ram_offset=0;
 194 #endif
 195
 196   int new_dynarec_hacks;
 197   int new_dynarec_did_compile;
 198   extern u_char restore_candidate[512];
 199   extern int cycle_count;
 200
 201   /* registers that may be allocated */
 202   /* 1-31 gpr */
 203 #define HIREG 32 // hi
 204 #define LOREG 33 // lo
 205 //#define FSREG 34 // FPU status (FCSR)
 206 #define CSREG 35 // Coprocessor status
 207 #define CCREG 36 // Cycle count
 208 #define INVCP 37 // Pointer to invalid_code
 209 //#define MMREG 38 // Pointer to memory_map
 210 //#define ROREG 39 // ram offset (if rdram!=0x80000000)
 211 #define TEMPREG 40
 212 #define FTEMP 40 // FPU temporary register
 213 #define PTEMP 41 // Prefetch temporary register
 214 //#define TLREG 42 // TLB mapping offset
 215 #define RHASH 43 // Return address hash
 216 #define RHTBL 44 // Return address hash table address
 217 #define RTEMP 45 // JR/JALR address register
 218 #define MAXREG 45
 219 #define AGEN1 46 // Address generation temporary register
 220 //#define AGEN2 47 // Address generation temporary register
 221 //#define MGEN1 48 // Maptable address generation temporary register
 222 //#define MGEN2 49 // Maptable address generation temporary register
 223 #define BTREG 50 // Branch target temporary register
 224
 225   /* instruction types */
 226 #define NOP 0     // No operation
 227 #define LOAD 1    // Load
 228 #define STORE 2   // Store
 229 #define LOADLR 3  // Unaligned load
 230 #define STORELR 4 // Unaligned store
 231 #define MOV 5     // Move
 232 #define ALU 6     // Arithmetic/logic
 233 #define MULTDIV 7 // Multiply/divide
 234 #define SHIFT 8   // Shift by register
 235 #define SHIFTIMM 9// Shift by immediate
 236 #define IMM16 10  // 16-bit immediate
 237 #define RJUMP 11  // Unconditional jump to register
 238 #define UJUMP 12  // Unconditional jump
 239 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 240 #define SJUMP 14  // Conditional branch (regimm format)
 241 #define COP0 15   // Coprocessor 0
 242 #define COP1 16   // Coprocessor 1
 243 #define C1LS 17   // Coprocessor 1 load/store
 244 //#define FJUMP 18  // Conditional branch (floating point)
 245 //#define FLOAT 19  // Floating point unit
 246 //#define FCONV 20  // Convert integer to float
 247 //#define FCOMP 21  // Floating point compare (sets FSREG)
 248 #define SYSCALL 22// SYSCALL
 249 #define OTHER 23  // Other
 250 #define SPAN 24   // Branch/delay slot spans 2 pages
 251 #define NI 25     // Not implemented
 252 #define HLECALL 26// PCSX fake opcodes for HLE
 253 #define COP2 27   // Coprocessor 2 move
 254 #define C2LS 28   // Coprocessor 2 load/store
 255 #define C2OP 29   // Coprocessor 2 operation
 256 #define INTCALL 30// Call interpreter to handle rare corner cases
 257
 258   /* branch codes */
 259 #define TAKEN 1
 260 #define NOTTAKEN 2
 261 #define NULLDS 3
 262
 263 // asm linkage
 264 int new_recompile_block(int addr);
 265 void *get_addr_ht(u_int vaddr);
 266 void invalidate_block(u_int block);
 267 void invalidate_addr(u_int addr);
 268 void remove_hash(int vaddr);
 269 void dyna_linker();
 270 void dyna_linker_ds();
 271 void verify_code();
 272 void verify_code_vm();
 273 void verify_code_ds();
 274 void cc_interrupt();
 275 void fp_exception();
 276 void fp_exception_ds();
 277 void jump_syscall_hle();
 278 void jump_hlecall();
 279 void jump_intcall();
 280 void new_dyna_leave();
 281
 282 // Needed by assembler
 283 static void wb_register(signed char r,signed char regmap[],uint64_t dirty);
 284 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty);
 285 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr);
 286 static void load_all_regs(signed char i_regmap[]);
 287 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 288 static void load_regs_entry(int t);
 289 static void load_all_consts(signed char regmap[],u_int dirty,int i);
 290
 291 static int verify_dirty(u_int *ptr);
 292 static int get_final_value(int hr, int i, int *value);
 293 static void add_stub(enum stub_type type, void *addr, void *retaddr,
 294   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e);
 295 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
 296   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist);
 297 static void add_to_linker(void *addr, u_int target, int ext);
 298 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override);
 299
 300 static void mprotect_w_x(void *start, void *end, int is_x)
 301 {
 302 #ifdef NO_WRITE_EXEC
 303   #if defined(VITA)
 304   // *Open* enables write on all memory that was
 305   // allocated by sceKernelAllocMemBlockForVM()?
 306   if (is_x)
 307     sceKernelCloseVMDomain();
 308   else
 309     sceKernelOpenVMDomain();
 310   #else
 311   u_long mstart = (u_long)start & ~4095ul;
 312   u_long mend = (u_long)end;
 313   if (mprotect((void *)mstart, mend - mstart,
 314                PROT_READ | (is_x ? PROT_EXEC : PROT_WRITE)) != 0)
 315     SysPrintf("mprotect(%c) failed: %s\n", is_x ? 'x' : 'w', strerror(errno));
 316   #endif
 317 #endif
 318 }
 319
 320 static void start_tcache_write(void *start, void *end)
 321 {
 322   mprotect_w_x(start, end, 0);
 323 }
 324
 325 static void end_tcache_write(void *start, void *end)
 326 {
 327 #ifdef __arm__
 328   size_t len = (char *)end - (char *)start;
 329   #if   defined(__BLACKBERRY_QNX__)
 330   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);
 331   #elif defined(__MACH__)
 332   sys_cache_control(kCacheFunctionPrepareForExecution, start, len);
 333   #elif defined(VITA)
 334   sceKernelSyncVMDomain(sceBlock, start, len);
 335   #elif defined(_3DS)
 336   ctr_flush_invalidate_cache();
 337   #else
 338   __clear_cache(start, end);
 339   #endif
 340   (void)len;
 341 #endif
 342
 343   mprotect_w_x(start, end, 1);
 344 }
 345
 346 static void *start_block(void)
 347 {
 348   u_char *end = out + MAX_OUTPUT_BLOCK_SIZE;
 349   if (end > translation_cache + (1<<TARGET_SIZE_2))
 350     end = translation_cache + (1<<TARGET_SIZE_2);
 351   start_tcache_write(out, end);
 352   return out;
 353 }
 354
 355 static void end_block(void *start)
 356 {
 357   end_tcache_write(start, out);
 358 }
 359
 360 //#define DEBUG_CYCLE_COUNT 1
 361
 362 #define NO_CYCLE_PENALTY_THR 12
 363
 364 int cycle_multiplier; // 100 for 1.0
 365
 366 static int CLOCK_ADJUST(int x)
 367 {
 368   int s=(x>>31)|1;
 369   return (x * cycle_multiplier + s * 50) / 100;
 370 }
 371
 372 static u_int get_page(u_int vaddr)
 373 {
 374   u_int page=vaddr&~0xe0000000;
 375   if (page < 0x1000000)
 376     page &= ~0x0e00000; // RAM mirrors
 377   page>>=12;
 378   if(page>2048) page=2048+(page&2047);
 379   return page;
 380 }
 381
 382 // no virtual mem in PCSX
 383 static u_int get_vpage(u_int vaddr)
 384 {
 385   return get_page(vaddr);
 386 }
 387
 388 static struct ht_entry *hash_table_get(u_int vaddr)
 389 {
 390   return &hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 391 }
 392
 393 static void hash_table_add(struct ht_entry *ht_bin, u_int vaddr, void *tcaddr)
 394 {
 395   ht_bin->vaddr[1] = ht_bin->vaddr[0];
 396   ht_bin->tcaddr[1] = ht_bin->tcaddr[0];
 397   ht_bin->vaddr[0] = vaddr;
 398   ht_bin->tcaddr[0] = tcaddr;
 399 }
 400
 401 // some messy ari64's code, seems to rely on unsigned 32bit overflow
 402 static int doesnt_expire_soon(void *tcaddr)
 403 {
 404   u_int diff = (u_int)((u_char *)tcaddr - out) << (32-TARGET_SIZE_2);
 405   return diff > (u_int)(0x60000000 + (MAX_OUTPUT_BLOCK_SIZE << (32-TARGET_SIZE_2)));
 406 }
 407
 408 // Get address from virtual address
 409 // This is called from the recompiled JR/JALR instructions
 410 void *get_addr(u_int vaddr)
 411 {
 412   u_int page=get_page(vaddr);
 413   u_int vpage=get_vpage(vaddr);
 414   struct ll_entry *head;
 415   //printf("TRACE: count=%d next=%d (get_addr %x,page %d)\n",Count,next_interupt,vaddr,page);
 416   head=jump_in[page];
 417   while(head!=NULL) {
 418     if(head->vaddr==vaddr) {
 419   //printf("TRACE: count=%d next=%d (get_addr match %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 420       hash_table_add(hash_table_get(vaddr), vaddr, head->addr);
 421       return head->addr;
 422     }
 423     head=head->next;
 424   }
 425   head=jump_dirty[vpage];
 426   while(head!=NULL) {
 427     if(head->vaddr==vaddr) {
 428       //printf("TRACE: count=%d next=%d (get_addr match dirty %x: %p)\n",Count,next_interupt,vaddr,head->addr);
 429       // Don't restore blocks which are about to expire from the cache
 430       if (doesnt_expire_soon(head->addr))
 431       if (verify_dirty(head->addr)) {
 432         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 433         invalid_code[vaddr>>12]=0;
 434         inv_code_start=inv_code_end=~0;
 435         if(vpage<2048) {
 436           restore_candidate[vpage>>3]|=1<<(vpage&7);
 437         }
 438         else restore_candidate[page>>3]|=1<<(page&7);
 439         struct ht_entry *ht_bin = hash_table_get(vaddr);
 440         if (ht_bin->vaddr[0] == vaddr)
 441           ht_bin->tcaddr[0] = head->addr; // Replace existing entry
 442         else
 443           hash_table_add(ht_bin, vaddr, head->addr);
 444
 445         return head->addr;
 446       }
 447     }
 448     head=head->next;
 449   }
 450   //printf("TRACE: count=%d next=%d (get_addr no-match %x)\n",Count,next_interupt,vaddr);
 451   int r=new_recompile_block(vaddr);
 452   if(r==0) return get_addr(vaddr);
 453   // Execute in unmapped page, generate pagefault execption
 454   Status|=2;
 455   Cause=(vaddr<<31)|0x8;
 456   EPC=(vaddr&1)?vaddr-5:vaddr;
 457   BadVAddr=(vaddr&~1);
 458   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 459   EntryHi=BadVAddr&0xFFFFE000;
 460   return get_addr_ht(0x80000000);
 461 }
 462 // Look up address in hash table first
 463 void *get_addr_ht(u_int vaddr)
 464 {
 465   //printf("TRACE: count=%d next=%d (get_addr_ht %x)\n",Count,next_interupt,vaddr);
 466   const struct ht_entry *ht_bin = hash_table_get(vaddr);
 467   if (ht_bin->vaddr[0] == vaddr) return ht_bin->tcaddr[0];
 468   if (ht_bin->vaddr[1] == vaddr) return ht_bin->tcaddr[1];
 469   return get_addr(vaddr);
 470 }
 471
 472 void clear_all_regs(signed char regmap[])
 473 {
 474   int hr;
 475   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 476 }
 477
 478 signed char get_reg(signed char regmap[],int r)
 479 {
 480   int hr;
 481   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 482   return -1;
 483 }
 484
 485 // Find a register that is available for two consecutive cycles
 486 signed char get_reg2(signed char regmap1[],signed char regmap2[],int r)
 487 {
 488   int hr;
 489   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 490   return -1;
 491 }
 492
 493 int count_free_regs(signed char regmap[])
 494 {
 495   int count=0;
 496   int hr;
 497   for(hr=0;hr<HOST_REGS;hr++)
 498   {
 499     if(hr!=EXCLUDE_REG) {
 500       if(regmap[hr]<0) count++;
 501     }
 502   }
 503   return count;
 504 }
 505
 506 void dirty_reg(struct regstat *cur,signed char reg)
 507 {
 508   int hr;
 509   if(!reg) return;
 510   for (hr=0;hr<HOST_REGS;hr++) {
 511     if((cur->regmap[hr]&63)==reg) {
 512       cur->dirty|=1<<hr;
 513     }
 514   }
 515 }
 516
 517 void set_const(struct regstat *cur,signed char reg,uint64_t value)
 518 {
 519   int hr;
 520   if(!reg) return;
 521   for (hr=0;hr<HOST_REGS;hr++) {
 522     if(cur->regmap[hr]==reg) {
 523       cur->isconst|=1<<hr;
 524       current_constmap[hr]=value;
 525     }
 526     else if((cur->regmap[hr]^64)==reg) {
 527       cur->isconst|=1<<hr;
 528       current_constmap[hr]=value>>32;
 529     }
 530   }
 531 }
 532
 533 void clear_const(struct regstat *cur,signed char reg)
 534 {
 535   int hr;
 536   if(!reg) return;
 537   for (hr=0;hr<HOST_REGS;hr++) {
 538     if((cur->regmap[hr]&63)==reg) {
 539       cur->isconst&=~(1<<hr);
 540     }
 541   }
 542 }
 543
 544 int is_const(struct regstat *cur,signed char reg)
 545 {
 546   int hr;
 547   if(reg<0) return 0;
 548   if(!reg) return 1;
 549   for (hr=0;hr<HOST_REGS;hr++) {
 550     if((cur->regmap[hr]&63)==reg) {
 551       return (cur->isconst>>hr)&1;
 552     }
 553   }
 554   return 0;
 555 }
 556 uint64_t get_const(struct regstat *cur,signed char reg)
 557 {
 558   int hr;
 559   if(!reg) return 0;
 560   for (hr=0;hr<HOST_REGS;hr++) {
 561     if(cur->regmap[hr]==reg) {
 562       return current_constmap[hr];
 563     }
 564   }
 565   SysPrintf("Unknown constant in r%d\n",reg);
 566   exit(1);
 567 }
 568
 569 // Least soon needed registers
 570 // Look at the next ten instructions and see which registers
 571 // will be used.  Try not to reallocate these.
 572 void lsn(u_char hsn[], int i, int *preferred_reg)
 573 {
 574   int j;
 575   int b=-1;
 576   for(j=0;j<9;j++)
 577   {
 578     if(i+j>=slen) {
 579       j=slen-i-1;
 580       break;
 581     }
 582     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 583     {
 584       // Don't go past an unconditonal jump
 585       j++;
 586       break;
 587     }
 588   }
 589   for(;j>=0;j--)
 590   {
 591     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 592     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 593     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 594     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 595     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 596       // Stores can allocate zero
 597       hsn[rs1[i+j]]=j;
 598       hsn[rs2[i+j]]=j;
 599     }
 600     // On some architectures stores need invc_ptr
 601     #if defined(HOST_IMM8)
 602     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39 || (opcode[i+j]&0x3b)==0x3a) {
 603       hsn[INVCP]=j;
 604     }
 605     #endif
 606     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 607     {
 608       hsn[CCREG]=j;
 609       b=j;
 610     }
 611   }
 612   if(b>=0)
 613   {
 614     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 615     {
 616       // Follow first branch
 617       int t=(ba[i+b]-start)>>2;
 618       j=7-b;if(t+j>=slen) j=slen-t-1;
 619       for(;j>=0;j--)
 620       {
 621         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 622         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 623         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 624         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 625       }
 626     }
 627     // TODO: preferred register based on backward branch
 628   }
 629   // Delay slot should preferably not overwrite branch conditions or cycle count
 630   if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
 631     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 632     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 633     hsn[CCREG]=1;
 634     // ...or hash tables
 635     hsn[RHASH]=1;
 636     hsn[RHTBL]=1;
 637   }
 638   // Coprocessor load/store needs FTEMP, even if not declared
 639   if(itype[i]==C1LS||itype[i]==C2LS) {
 640     hsn[FTEMP]=0;
 641   }
 642   // Load L/R also uses FTEMP as a temporary register
 643   if(itype[i]==LOADLR) {
 644     hsn[FTEMP]=0;
 645   }
 646   // Also SWL/SWR/SDL/SDR
 647   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) {
 648     hsn[FTEMP]=0;
 649   }
 650   // Don't remove the miniht registers
 651   if(itype[i]==UJUMP||itype[i]==RJUMP)
 652   {
 653     hsn[RHASH]=0;
 654     hsn[RHTBL]=0;
 655   }
 656 }
 657
 658 // We only want to allocate registers if we're going to use them again soon
 659 int needed_again(int r, int i)
 660 {
 661   int j;
 662   int b=-1;
 663   int rn=10;
 664
 665   if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000))
 666   {
 667     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 668       return 0; // Don't need any registers if exiting the block
 669   }
 670   for(j=0;j<9;j++)
 671   {
 672     if(i+j>=slen) {
 673       j=slen-i-1;
 674       break;
 675     }
 676     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 677     {
 678       // Don't go past an unconditonal jump
 679       j++;
 680       break;
 681     }
 682     if(itype[i+j]==SYSCALL||itype[i+j]==HLECALL||itype[i+j]==INTCALL||((source[i+j]&0xfc00003f)==0x0d))
 683     {
 684       break;
 685     }
 686   }
 687   for(;j>=1;j--)
 688   {
 689     if(rs1[i+j]==r) rn=j;
 690     if(rs2[i+j]==r) rn=j;
 691     if((unneeded_reg[i+j]>>r)&1) rn=10;
 692     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP))
 693     {
 694       b=j;
 695     }
 696   }
 697   /*
 698   if(b>=0)
 699   {
 700     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 701     {
 702       // Follow first branch
 703       int o=rn;
 704       int t=(ba[i+b]-start)>>2;
 705       j=7-b;if(t+j>=slen) j=slen-t-1;
 706       for(;j>=0;j--)
 707       {
 708         if(!((unneeded_reg[t+j]>>r)&1)) {
 709           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 710           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 711         }
 712         else rn=o;
 713       }
 714     }
 715   }*/
 716   if(rn<10) return 1;
 717   (void)b;
 718   return 0;
 719 }
 720
 721 // Try to match register allocations at the end of a loop with those
 722 // at the beginning
 723 int loop_reg(int i, int r, int hr)
 724 {
 725   int j,k;
 726   for(j=0;j<9;j++)
 727   {
 728     if(i+j>=slen) {
 729       j=slen-i-1;
 730       break;
 731     }
 732     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 733     {
 734       // Don't go past an unconditonal jump
 735       j++;
 736       break;
 737     }
 738   }
 739   k=0;
 740   if(i>0){
 741     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)
 742       k--;
 743   }
 744   for(;k<j;k++)
 745   {
 746     assert(r < 64);
 747     if((unneeded_reg[i+k]>>r)&1) return hr;
 748     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP))
 749     {
 750       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 751       {
 752         int t=(ba[i+k]-start)>>2;
 753         int reg=get_reg(regs[t].regmap_entry,r);
 754         if(reg>=0) return reg;
 755         //reg=get_reg(regs[t+1].regmap_entry,r);
 756         //if(reg>=0) return reg;
 757       }
 758     }
 759   }
 760   return hr;
 761 }
 762
 763
 764 // Allocate every register, preserving source/target regs
 765 void alloc_all(struct regstat *cur,int i)
 766 {
 767   int hr;
 768
 769   for(hr=0;hr<HOST_REGS;hr++) {
 770     if(hr!=EXCLUDE_REG) {
 771       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 772          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 773       {
 774         cur->regmap[hr]=-1;
 775         cur->dirty&=~(1<<hr);
 776       }
 777       // Don't need zeros
 778       if((cur->regmap[hr]&63)==0)
 779       {
 780         cur->regmap[hr]=-1;
 781         cur->dirty&=~(1<<hr);
 782       }
 783     }
 784   }
 785 }
 786
 787 #ifdef DRC_DBG
 788 extern void gen_interupt();
 789 extern void do_insn_cmp();
 790 #define FUNCNAME(f) { (intptr_t)f, " " #f }
 791 static const struct {
 792   intptr_t addr;
 793   const char *name;
 794 } function_names[] = {
 795   FUNCNAME(cc_interrupt),
 796   FUNCNAME(gen_interupt),
 797   FUNCNAME(get_addr_ht),
 798   FUNCNAME(get_addr),
 799   FUNCNAME(jump_handler_read8),
 800   FUNCNAME(jump_handler_read16),
 801   FUNCNAME(jump_handler_read32),
 802   FUNCNAME(jump_handler_write8),
 803   FUNCNAME(jump_handler_write16),
 804   FUNCNAME(jump_handler_write32),
 805   FUNCNAME(invalidate_addr),
 806   FUNCNAME(verify_code_vm),
 807   FUNCNAME(verify_code),
 808   FUNCNAME(jump_hlecall),
 809   FUNCNAME(jump_syscall_hle),
 810   FUNCNAME(new_dyna_leave),
 811   FUNCNAME(pcsx_mtc0),
 812   FUNCNAME(pcsx_mtc0_ds),
 813   FUNCNAME(do_insn_cmp),
 814 };
 815
 816 static const char *func_name(intptr_t a)
 817 {
 818   int i;
 819   for (i = 0; i < sizeof(function_names)/sizeof(function_names[0]); i++)
 820     if (function_names[i].addr == a)
 821       return function_names[i].name;
 822   return "";
 823 }
 824 #else
 825 #define func_name(x) ""
 826 #endif
 827
 828 #ifdef __i386__
 829 #include "assem_x86.c"
 830 #endif
 831 #ifdef __x86_64__
 832 #include "assem_x64.c"
 833 #endif
 834 #ifdef __arm__
 835 #include "assem_arm.c"
 836 #endif
 837
 838 // Add virtual address mapping to linked list
 839 void ll_add(struct ll_entry **head,int vaddr,void *addr)
 840 {
 841   struct ll_entry *new_entry;
 842   new_entry=malloc(sizeof(struct ll_entry));
 843   assert(new_entry!=NULL);
 844   new_entry->vaddr=vaddr;
 845   new_entry->reg_sv_flags=0;
 846   new_entry->addr=addr;
 847   new_entry->next=*head;
 848   *head=new_entry;
 849 }
 850
 851 void ll_add_flags(struct ll_entry **head,int vaddr,u_int reg_sv_flags,void *addr)
 852 {
 853   ll_add(head,vaddr,addr);
 854   (*head)->reg_sv_flags=reg_sv_flags;
 855 }
 856
 857 // Check if an address is already compiled
 858 // but don't return addresses which are about to expire from the cache
 859 void *check_addr(u_int vaddr)
 860 {
 861   struct ht_entry *ht_bin = hash_table_get(vaddr);
 862   size_t i;
 863   for (i = 0; i < ARRAY_SIZE(ht_bin->vaddr); i++) {
 864     if (ht_bin->vaddr[i] == vaddr)
 865       if (doesnt_expire_soon((u_char *)ht_bin->tcaddr[i] - MAX_OUTPUT_BLOCK_SIZE))
 866         if (isclean(ht_bin->tcaddr[i]))
 867           return ht_bin->tcaddr[i];
 868   }
 869   u_int page=get_page(vaddr);
 870   struct ll_entry *head;
 871   head=jump_in[page];
 872   while (head != NULL) {
 873     if (head->vaddr == vaddr) {
 874       if (doesnt_expire_soon(head->addr)) {
 875         // Update existing entry with current address
 876         if (ht_bin->vaddr[0] == vaddr) {
 877           ht_bin->tcaddr[0] = head->addr;
 878           return head->addr;
 879         }
 880         if (ht_bin->vaddr[1] == vaddr) {
 881           ht_bin->tcaddr[1] = head->addr;
 882           return head->addr;
 883         }
 884         // Insert into hash table with low priority.
 885         // Don't evict existing entries, as they are probably
 886         // addresses that are being accessed frequently.
 887         if (ht_bin->vaddr[0] == -1) {
 888           ht_bin->vaddr[0] = vaddr;
 889           ht_bin->tcaddr[0] = head->addr;
 890         }
 891         else if (ht_bin->vaddr[1] == -1) {
 892           ht_bin->vaddr[1] = vaddr;
 893           ht_bin->tcaddr[1] = head->addr;
 894         }
 895         return head->addr;
 896       }
 897     }
 898     head=head->next;
 899   }
 900   return 0;
 901 }
 902
 903 void remove_hash(int vaddr)
 904 {
 905   //printf("remove hash: %x\n",vaddr);
 906   struct ht_entry *ht_bin = hash_table_get(vaddr);
 907   if (ht_bin->vaddr[1] == vaddr) {
 908     ht_bin->vaddr[1] = -1;
 909     ht_bin->tcaddr[1] = NULL;
 910   }
 911   if (ht_bin->vaddr[0] == vaddr) {
 912     ht_bin->vaddr[0] = ht_bin->vaddr[1];
 913     ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
 914     ht_bin->vaddr[1] = -1;
 915     ht_bin->tcaddr[1] = NULL;
 916   }
 917 }
 918
 919 void ll_remove_matching_addrs(struct ll_entry **head,uintptr_t addr,int shift)
 920 {
 921   struct ll_entry *next;
 922   while(*head) {
 923     if(((uintptr_t)((*head)->addr)>>shift)==(addr>>shift) ||
 924        ((uintptr_t)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
 925     {
 926       inv_debug("EXP: Remove pointer to %p (%x)\n",(*head)->addr,(*head)->vaddr);
 927       remove_hash((*head)->vaddr);
 928       next=(*head)->next;
 929       free(*head);
 930       *head=next;
 931     }
 932     else
 933     {
 934       head=&((*head)->next);
 935     }
 936   }
 937 }
 938
 939 // Remove all entries from linked list
 940 void ll_clear(struct ll_entry **head)
 941 {
 942   struct ll_entry *cur;
 943   struct ll_entry *next;
 944   if((cur=*head)) {
 945     *head=0;
 946     while(cur) {
 947       next=cur->next;
 948       free(cur);
 949       cur=next;
 950     }
 951   }
 952 }
 953
 954 // Dereference the pointers and remove if it matches
 955 static void ll_kill_pointers(struct ll_entry *head,uintptr_t addr,int shift)
 956 {
 957   while(head) {
 958     uintptr_t ptr = (uintptr_t)get_pointer(head->addr);
 959     inv_debug("EXP: Lookup pointer to %lx at %p (%x)\n",(long)ptr,head->addr,head->vaddr);
 960     if(((ptr>>shift)==(addr>>shift)) ||
 961        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
 962     {
 963       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
 964       void *host_addr=find_extjump_insn(head->addr);
 965       #ifdef __arm__
 966         mark_clear_cache(host_addr);
 967       #endif
 968       set_jump_target(host_addr, head->addr);
 969     }
 970     head=head->next;
 971   }
 972 }
 973
 974 // This is called when we write to a compiled block (see do_invstub)
 975 void invalidate_page(u_int page)
 976 {
 977   struct ll_entry *head;
 978   struct ll_entry *next;
 979   head=jump_in[page];
 980   jump_in[page]=0;
 981   while(head!=NULL) {
 982     inv_debug("INVALIDATE: %x\n",head->vaddr);
 983     remove_hash(head->vaddr);
 984     next=head->next;
 985     free(head);
 986     head=next;
 987   }
 988   head=jump_out[page];
 989   jump_out[page]=0;
 990   while(head!=NULL) {
 991     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
 992     void *host_addr=find_extjump_insn(head->addr);
 993     #ifdef __arm__
 994       mark_clear_cache(host_addr);
 995     #endif
 996     set_jump_target(host_addr, head->addr);
 997     next=head->next;
 998     free(head);
 999     head=next;
1000   }
1001 }
1002
1003 static void invalidate_block_range(u_int block, u_int first, u_int last)
1004 {
1005   u_int page=get_page(block<<12);
1006   //printf("first=%d last=%d\n",first,last);
1007   invalidate_page(page);
1008   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1009   assert(last<page+5);
1010   // Invalidate the adjacent pages if a block crosses a 4K boundary
1011   while(first<page) {
1012     invalidate_page(first);
1013     first++;
1014   }
1015   for(first=page+1;first<last;first++) {
1016     invalidate_page(first);
1017   }
1018   #ifdef __arm__
1019     do_clear_cache();
1020   #endif
1021
1022   // Don't trap writes
1023   invalid_code[block]=1;
1024
1025   #ifdef USE_MINI_HT
1026   memset(mini_ht,-1,sizeof(mini_ht));
1027   #endif
1028 }
1029
1030 void invalidate_block(u_int block)
1031 {
1032   u_int page=get_page(block<<12);
1033   u_int vpage=get_vpage(block<<12);
1034   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1035   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1036   u_int first,last;
1037   first=last=page;
1038   struct ll_entry *head;
1039   head=jump_dirty[vpage];
1040   //printf("page=%d vpage=%d\n",page,vpage);
1041   while(head!=NULL) {
1042     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1043       u_char *start, *end;
1044       get_bounds(head->addr, &start, &end);
1045       //printf("start: %p end: %p\n", start, end);
1046       if (page < 2048 && start >= rdram && end < rdram+RAM_SIZE) {
1047         if (((start-rdram)>>12) <= page && ((end-1-rdram)>>12) >= page) {
1048           if ((((start-rdram)>>12)&2047) < first) first = ((start-rdram)>>12)&2047;
1049           if ((((end-1-rdram)>>12)&2047) > last)  last = ((end-1-rdram)>>12)&2047;
1050         }
1051       }
1052     }
1053     head=head->next;
1054   }
1055   invalidate_block_range(block,first,last);
1056 }
1057
1058 void invalidate_addr(u_int addr)
1059 {
1060   //static int rhits;
1061   // this check is done by the caller
1062   //if (inv_code_start<=addr&&addr<=inv_code_end) { rhits++; return; }
1063   u_int page=get_vpage(addr);
1064   if(page<2048) { // RAM
1065     struct ll_entry *head;
1066     u_int addr_min=~0, addr_max=0;
1067     u_int mask=RAM_SIZE-1;
1068     u_int addr_main=0x80000000|(addr&mask);
1069     int pg1;
1070     inv_code_start=addr_main&~0xfff;
1071     inv_code_end=addr_main|0xfff;
1072     pg1=page;
1073     if (pg1>0) {
1074       // must check previous page too because of spans..
1075       pg1--;
1076       inv_code_start-=0x1000;
1077     }
1078     for(;pg1<=page;pg1++) {
1079       for(head=jump_dirty[pg1];head!=NULL;head=head->next) {
1080         u_char *start_h, *end_h;
1081         u_int start, end;
1082         get_bounds(head->addr, &start_h, &end_h);
1083         start = (uintptr_t)start_h - ram_offset;
1084         end = (uintptr_t)end_h - ram_offset;
1085         if(start<=addr_main&&addr_main<end) {
1086           if(start<addr_min) addr_min=start;
1087           if(end>addr_max) addr_max=end;
1088         }
1089         else if(addr_main<start) {
1090           if(start<inv_code_end)
1091             inv_code_end=start-1;
1092         }
1093         else {
1094           if(end>inv_code_start)
1095             inv_code_start=end;
1096         }
1097       }
1098     }
1099     if (addr_min!=~0) {
1100       inv_debug("INV ADDR: %08x hit %08x-%08x\n", addr, addr_min, addr_max);
1101       inv_code_start=inv_code_end=~0;
1102       invalidate_block_range(addr>>12,(addr_min&mask)>>12,(addr_max&mask)>>12);
1103       return;
1104     }
1105     else {
1106       inv_code_start=(addr&~mask)|(inv_code_start&mask);
1107       inv_code_end=(addr&~mask)|(inv_code_end&mask);
1108       inv_debug("INV ADDR: %08x miss, inv %08x-%08x, sk %d\n", addr, inv_code_start, inv_code_end, 0);
1109       return;
1110     }
1111   }
1112   invalidate_block(addr>>12);
1113 }
1114
1115 // This is called when loading a save state.
1116 // Anything could have changed, so invalidate everything.
1117 void invalidate_all_pages()
1118 {
1119   u_int page;
1120   for(page=0;page<4096;page++)
1121     invalidate_page(page);
1122   for(page=0;page<1048576;page++)
1123     if(!invalid_code[page]) {
1124       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1125       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1126     }
1127   #ifdef USE_MINI_HT
1128   memset(mini_ht,-1,sizeof(mini_ht));
1129   #endif
1130 }
1131
1132 // Add an entry to jump_out after making a link
1133 void add_link(u_int vaddr,void *src)
1134 {
1135   u_int page=get_page(vaddr);
1136   inv_debug("add_link: %p -> %x (%d)\n",src,vaddr,page);
1137   int *ptr=(int *)(src+4);
1138   assert((*ptr&0x0fff0000)==0x059f0000);
1139   (void)ptr;
1140   ll_add(jump_out+page,vaddr,src);
1141   //void *ptr=get_pointer(src);
1142   //inv_debug("add_link: Pointer is to %p\n",ptr);
1143 }
1144
1145 // If a code block was found to be unmodified (bit was set in
1146 // restore_candidate) and it remains unmodified (bit is clear
1147 // in invalid_code) then move the entries for that 4K page from
1148 // the dirty list to the clean list.
1149 void clean_blocks(u_int page)
1150 {
1151   struct ll_entry *head;
1152   inv_debug("INV: clean_blocks page=%d\n",page);
1153   head=jump_dirty[page];
1154   while(head!=NULL) {
1155     if(!invalid_code[head->vaddr>>12]) {
1156       // Don't restore blocks which are about to expire from the cache
1157       if (doesnt_expire_soon(head->addr)) {
1158         if(verify_dirty(head->addr)) {
1159           u_char *start, *end;
1160           //printf("Possibly Restore %x (%p)\n",head->vaddr, head->addr);
1161           u_int i;
1162           u_int inv=0;
1163           get_bounds(head->addr, &start, &end);
1164           if (start - rdram < RAM_SIZE) {
1165             for (i = (start-rdram+0x80000000)>>12; i <= (end-1-rdram+0x80000000)>>12; i++) {
1166               inv|=invalid_code[i];
1167             }
1168           }
1169           else if((signed int)head->vaddr>=(signed int)0x80000000+RAM_SIZE) {
1170             inv=1;
1171           }
1172           if(!inv) {
1173             void *clean_addr = get_clean_addr(head->addr);
1174             if (doesnt_expire_soon(clean_addr)) {
1175               u_int ppage=page;
1176               inv_debug("INV: Restored %x (%p/%p)\n",head->vaddr, head->addr, clean_addr);
1177               //printf("page=%x, addr=%x\n",page,head->vaddr);
1178               //assert(head->vaddr>>12==(page|0x80000));
1179               ll_add_flags(jump_in+ppage,head->vaddr,head->reg_sv_flags,clean_addr);
1180               struct ht_entry *ht_bin = hash_table_get(head->vaddr);
1181               if (ht_bin->vaddr[0] == head->vaddr)
1182                 ht_bin->tcaddr[0] = clean_addr; // Replace existing entry
1183               if (ht_bin->vaddr[1] == head->vaddr)
1184                 ht_bin->tcaddr[1] = clean_addr; // Replace existing entry
1185             }
1186           }
1187         }
1188       }
1189     }
1190     head=head->next;
1191   }
1192 }
1193
1194 /* Register allocation */
1195
1196 // Note: registers are allocated clean (unmodified state)
1197 // if you intend to modify the register, you must call dirty_reg().
1198 static void alloc_reg(struct regstat *cur,int i,signed char reg)
1199 {
1200   int r,hr;
1201   int preferred_reg = (reg&7);
1202   if(reg==CCREG) preferred_reg=HOST_CCREG;
1203   if(reg==PTEMP||reg==FTEMP) preferred_reg=12;
1204
1205   // Don't allocate unused registers
1206   if((cur->u>>reg)&1) return;
1207
1208   // see if it's already allocated
1209   for(hr=0;hr<HOST_REGS;hr++)
1210   {
1211     if(cur->regmap[hr]==reg) return;
1212   }
1213
1214   // Keep the same mapping if the register was already allocated in a loop
1215   preferred_reg = loop_reg(i,reg,preferred_reg);
1216
1217   // Try to allocate the preferred register
1218   if(cur->regmap[preferred_reg]==-1) {
1219     cur->regmap[preferred_reg]=reg;
1220     cur->dirty&=~(1<<preferred_reg);
1221     cur->isconst&=~(1<<preferred_reg);
1222     return;
1223   }
1224   r=cur->regmap[preferred_reg];
1225   assert(r < 64);
1226   if((cur->u>>r)&1) {
1227     cur->regmap[preferred_reg]=reg;
1228     cur->dirty&=~(1<<preferred_reg);
1229     cur->isconst&=~(1<<preferred_reg);
1230     return;
1231   }
1232
1233   // Clear any unneeded registers
1234   // We try to keep the mapping consistent, if possible, because it
1235   // makes branches easier (especially loops).  So we try to allocate
1236   // first (see above) before removing old mappings.  If this is not
1237   // possible then go ahead and clear out the registers that are no
1238   // longer needed.
1239   for(hr=0;hr<HOST_REGS;hr++)
1240   {
1241     r=cur->regmap[hr];
1242     if(r>=0) {
1243       assert(r < 64);
1244       if((cur->u>>r)&1) {cur->regmap[hr]=-1;break;}
1245     }
1246   }
1247   // Try to allocate any available register, but prefer
1248   // registers that have not been used recently.
1249   if(i>0) {
1250     for(hr=0;hr<HOST_REGS;hr++) {
1251       if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1252         if(regs[i-1].regmap[hr]!=rs1[i-1]&&regs[i-1].regmap[hr]!=rs2[i-1]&&regs[i-1].regmap[hr]!=rt1[i-1]&&regs[i-1].regmap[hr]!=rt2[i-1]) {
1253           cur->regmap[hr]=reg;
1254           cur->dirty&=~(1<<hr);
1255           cur->isconst&=~(1<<hr);
1256           return;
1257         }
1258       }
1259     }
1260   }
1261   // Try to allocate any available register
1262   for(hr=0;hr<HOST_REGS;hr++) {
1263     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1264       cur->regmap[hr]=reg;
1265       cur->dirty&=~(1<<hr);
1266       cur->isconst&=~(1<<hr);
1267       return;
1268     }
1269   }
1270
1271   // Ok, now we have to evict someone
1272   // Pick a register we hopefully won't need soon
1273   u_char hsn[MAXREG+1];
1274   memset(hsn,10,sizeof(hsn));
1275   int j;
1276   lsn(hsn,i,&preferred_reg);
1277   //printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",cur->regmap[0],cur->regmap[1],cur->regmap[2],cur->regmap[3],cur->regmap[5],cur->regmap[6],cur->regmap[7]);
1278   //printf("hsn(%x): %d %d %d %d %d %d %d\n",start+i*4,hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1279   if(i>0) {
1280     // Don't evict the cycle count at entry points, otherwise the entry
1281     // stub will have to write it.
1282     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1283     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1284     for(j=10;j>=3;j--)
1285     {
1286       // Alloc preferred register if available
1287       if(hsn[r=cur->regmap[preferred_reg]&63]==j) {
1288         for(hr=0;hr<HOST_REGS;hr++) {
1289           // Evict both parts of a 64-bit register
1290           if((cur->regmap[hr]&63)==r) {
1291             cur->regmap[hr]=-1;
1292             cur->dirty&=~(1<<hr);
1293             cur->isconst&=~(1<<hr);
1294           }
1295         }
1296         cur->regmap[preferred_reg]=reg;
1297         return;
1298       }
1299       for(r=1;r<=MAXREG;r++)
1300       {
1301         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1302           for(hr=0;hr<HOST_REGS;hr++) {
1303             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1304               if(cur->regmap[hr]==r+64) {
1305                 cur->regmap[hr]=reg;
1306                 cur->dirty&=~(1<<hr);
1307                 cur->isconst&=~(1<<hr);
1308                 return;
1309               }
1310             }
1311           }
1312           for(hr=0;hr<HOST_REGS;hr++) {
1313             if(hr!=HOST_CCREG||j<hsn[CCREG]) {
1314               if(cur->regmap[hr]==r) {
1315                 cur->regmap[hr]=reg;
1316                 cur->dirty&=~(1<<hr);
1317                 cur->isconst&=~(1<<hr);
1318                 return;
1319               }
1320             }
1321           }
1322         }
1323       }
1324     }
1325   }
1326   for(j=10;j>=0;j--)
1327   {
1328     for(r=1;r<=MAXREG;r++)
1329     {
1330       if(hsn[r]==j) {
1331         for(hr=0;hr<HOST_REGS;hr++) {
1332           if(cur->regmap[hr]==r+64) {
1333             cur->regmap[hr]=reg;
1334             cur->dirty&=~(1<<hr);
1335             cur->isconst&=~(1<<hr);
1336             return;
1337           }
1338         }
1339         for(hr=0;hr<HOST_REGS;hr++) {
1340           if(cur->regmap[hr]==r) {
1341             cur->regmap[hr]=reg;
1342             cur->dirty&=~(1<<hr);
1343             cur->isconst&=~(1<<hr);
1344             return;
1345           }
1346         }
1347       }
1348     }
1349   }
1350   SysPrintf("This shouldn't happen (alloc_reg)");exit(1);
1351 }
1352
1353 // Allocate a temporary register.  This is done without regard to
1354 // dirty status or whether the register we request is on the unneeded list
1355 // Note: This will only allocate one register, even if called multiple times
1356 static void alloc_reg_temp(struct regstat *cur,int i,signed char reg)
1357 {
1358   int r,hr;
1359   int preferred_reg = -1;
1360
1361   // see if it's already allocated
1362   for(hr=0;hr<HOST_REGS;hr++)
1363   {
1364     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==reg) return;
1365   }
1366
1367   // Try to allocate any available register
1368   for(hr=HOST_REGS-1;hr>=0;hr--) {
1369     if(hr!=EXCLUDE_REG&&cur->regmap[hr]==-1) {
1370       cur->regmap[hr]=reg;
1371       cur->dirty&=~(1<<hr);
1372       cur->isconst&=~(1<<hr);
1373       return;
1374     }
1375   }
1376
1377   // Find an unneeded register
1378   for(hr=HOST_REGS-1;hr>=0;hr--)
1379   {
1380     r=cur->regmap[hr];
1381     if(r>=0) {
1382       assert(r < 64);
1383       if((cur->u>>r)&1) {
1384         if(i==0||((unneeded_reg[i-1]>>r)&1)) {
1385           cur->regmap[hr]=reg;
1386           cur->dirty&=~(1<<hr);
1387           cur->isconst&=~(1<<hr);
1388           return;
1389         }
1390       }
1391     }
1392   }
1393
1394   // Ok, now we have to evict someone
1395   // Pick a register we hopefully won't need soon
1396   // TODO: we might want to follow unconditional jumps here
1397   // TODO: get rid of dupe code and make this into a function
1398   u_char hsn[MAXREG+1];
1399   memset(hsn,10,sizeof(hsn));
1400   int j;
1401   lsn(hsn,i,&preferred_reg);
1402   //printf("hsn: %d %d %d %d %d %d %d\n",hsn[cur->regmap[0]&63],hsn[cur->regmap[1]&63],hsn[cur->regmap[2]&63],hsn[cur->regmap[3]&63],hsn[cur->regmap[5]&63],hsn[cur->regmap[6]&63],hsn[cur->regmap[7]&63]);
1403   if(i>0) {
1404     // Don't evict the cycle count at entry points, otherwise the entry
1405     // stub will have to write it.
1406     if(bt[i]&&hsn[CCREG]>2) hsn[CCREG]=2;
1407     if(i>1&&hsn[CCREG]>2&&(itype[i-2]==RJUMP||itype[i-2]==UJUMP||itype[i-2]==CJUMP||itype[i-2]==SJUMP)) hsn[CCREG]=2;
1408     for(j=10;j>=3;j--)
1409     {
1410       for(r=1;r<=MAXREG;r++)
1411       {
1412         if(hsn[r]==j&&r!=rs1[i-1]&&r!=rs2[i-1]&&r!=rt1[i-1]&&r!=rt2[i-1]) {
1413           for(hr=0;hr<HOST_REGS;hr++) {
1414             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1415               if(cur->regmap[hr]==r+64) {
1416                 cur->regmap[hr]=reg;
1417                 cur->dirty&=~(1<<hr);
1418                 cur->isconst&=~(1<<hr);
1419                 return;
1420               }
1421             }
1422           }
1423           for(hr=0;hr<HOST_REGS;hr++) {
1424             if(hr!=HOST_CCREG||hsn[CCREG]>2) {
1425               if(cur->regmap[hr]==r) {
1426                 cur->regmap[hr]=reg;
1427                 cur->dirty&=~(1<<hr);
1428                 cur->isconst&=~(1<<hr);
1429                 return;
1430               }
1431             }
1432           }
1433         }
1434       }
1435     }
1436   }
1437   for(j=10;j>=0;j--)
1438   {
1439     for(r=1;r<=MAXREG;r++)
1440     {
1441       if(hsn[r]==j) {
1442         for(hr=0;hr<HOST_REGS;hr++) {
1443           if(cur->regmap[hr]==r+64) {
1444             cur->regmap[hr]=reg;
1445             cur->dirty&=~(1<<hr);
1446             cur->isconst&=~(1<<hr);
1447             return;
1448           }
1449         }
1450         for(hr=0;hr<HOST_REGS;hr++) {
1451           if(cur->regmap[hr]==r) {
1452             cur->regmap[hr]=reg;
1453             cur->dirty&=~(1<<hr);
1454             cur->isconst&=~(1<<hr);
1455             return;
1456           }
1457         }
1458       }
1459     }
1460   }
1461   SysPrintf("This shouldn't happen");exit(1);
1462 }
1463
1464 static void mov_alloc(struct regstat *current,int i)
1465 {
1466   // Note: Don't need to actually alloc the source registers
1467   //alloc_reg(current,i,rs1[i]);
1468   alloc_reg(current,i,rt1[i]);
1469
1470   clear_const(current,rs1[i]);
1471   clear_const(current,rt1[i]);
1472   dirty_reg(current,rt1[i]);
1473 }
1474
1475 static void shiftimm_alloc(struct regstat *current,int i)
1476 {
1477   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1478   {
1479     if(rt1[i]) {
1480       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1481       else lt1[i]=rs1[i];
1482       alloc_reg(current,i,rt1[i]);
1483       dirty_reg(current,rt1[i]);
1484       if(is_const(current,rs1[i])) {
1485         int v=get_const(current,rs1[i]);
1486         if(opcode2[i]==0x00) set_const(current,rt1[i],v<<imm[i]);
1487         if(opcode2[i]==0x02) set_const(current,rt1[i],(u_int)v>>imm[i]);
1488         if(opcode2[i]==0x03) set_const(current,rt1[i],v>>imm[i]);
1489       }
1490       else clear_const(current,rt1[i]);
1491     }
1492   }
1493   else
1494   {
1495     clear_const(current,rs1[i]);
1496     clear_const(current,rt1[i]);
1497   }
1498
1499   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1500   {
1501     assert(0);
1502   }
1503   if(opcode2[i]==0x3c) // DSLL32
1504   {
1505     assert(0);
1506   }
1507   if(opcode2[i]==0x3e) // DSRL32
1508   {
1509     assert(0);
1510   }
1511   if(opcode2[i]==0x3f) // DSRA32
1512   {
1513     assert(0);
1514   }
1515 }
1516
1517 static void shift_alloc(struct regstat *current,int i)
1518 {
1519   if(rt1[i]) {
1520     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1521     {
1522       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1523       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1524       alloc_reg(current,i,rt1[i]);
1525       if(rt1[i]==rs2[i]) {
1526         alloc_reg_temp(current,i,-1);
1527         minimum_free_regs[i]=1;
1528       }
1529     } else { // DSLLV/DSRLV/DSRAV
1530       assert(0);
1531     }
1532     clear_const(current,rs1[i]);
1533     clear_const(current,rs2[i]);
1534     clear_const(current,rt1[i]);
1535     dirty_reg(current,rt1[i]);
1536   }
1537 }
1538
1539 static void alu_alloc(struct regstat *current,int i)
1540 {
1541   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1542     if(rt1[i]) {
1543       if(rs1[i]&&rs2[i]) {
1544         alloc_reg(current,i,rs1[i]);
1545         alloc_reg(current,i,rs2[i]);
1546       }
1547       else {
1548         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1549         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1550       }
1551       alloc_reg(current,i,rt1[i]);
1552     }
1553   }
1554   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1555     if(rt1[i]) {
1556       alloc_reg(current,i,rs1[i]);
1557       alloc_reg(current,i,rs2[i]);
1558       alloc_reg(current,i,rt1[i]);
1559     }
1560   }
1561   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1562     if(rt1[i]) {
1563       if(rs1[i]&&rs2[i]) {
1564         alloc_reg(current,i,rs1[i]);
1565         alloc_reg(current,i,rs2[i]);
1566       }
1567       else
1568       {
1569         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1570         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1571       }
1572       alloc_reg(current,i,rt1[i]);
1573     }
1574   }
1575   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1576     assert(0);
1577   }
1578   clear_const(current,rs1[i]);
1579   clear_const(current,rs2[i]);
1580   clear_const(current,rt1[i]);
1581   dirty_reg(current,rt1[i]);
1582 }
1583
1584 static void imm16_alloc(struct regstat *current,int i)
1585 {
1586   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1587   else lt1[i]=rs1[i];
1588   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1589   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1590     assert(0);
1591   }
1592   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1593     clear_const(current,rs1[i]);
1594     clear_const(current,rt1[i]);
1595   }
1596   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1597     if(is_const(current,rs1[i])) {
1598       int v=get_const(current,rs1[i]);
1599       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1600       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1601       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1602     }
1603     else clear_const(current,rt1[i]);
1604   }
1605   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1606     if(is_const(current,rs1[i])) {
1607       int v=get_const(current,rs1[i]);
1608       set_const(current,rt1[i],v+imm[i]);
1609     }
1610     else clear_const(current,rt1[i]);
1611   }
1612   else {
1613     set_const(current,rt1[i],((long long)((short)imm[i]))<<16); // LUI
1614   }
1615   dirty_reg(current,rt1[i]);
1616 }
1617
1618 static void load_alloc(struct regstat *current,int i)
1619 {
1620   clear_const(current,rt1[i]);
1621   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1622   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1623   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1624   if(rt1[i]&&!((current->u>>rt1[i])&1)) {
1625     alloc_reg(current,i,rt1[i]);
1626     assert(get_reg(current->regmap,rt1[i])>=0);
1627     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1628     {
1629       assert(0);
1630     }
1631     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1632     {
1633       assert(0);
1634     }
1635     dirty_reg(current,rt1[i]);
1636     // LWL/LWR need a temporary register for the old value
1637     if(opcode[i]==0x22||opcode[i]==0x26)
1638     {
1639       alloc_reg(current,i,FTEMP);
1640       alloc_reg_temp(current,i,-1);
1641       minimum_free_regs[i]=1;
1642     }
1643   }
1644   else
1645   {
1646     // Load to r0 or unneeded register (dummy load)
1647     // but we still need a register to calculate the address
1648     if(opcode[i]==0x22||opcode[i]==0x26)
1649     {
1650       alloc_reg(current,i,FTEMP); // LWL/LWR need another temporary
1651     }
1652     alloc_reg_temp(current,i,-1);
1653     minimum_free_regs[i]=1;
1654     if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1655     {
1656       assert(0);
1657     }
1658   }
1659 }
1660
1661 void store_alloc(struct regstat *current,int i)
1662 {
1663   clear_const(current,rs2[i]);
1664   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1665   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1666   alloc_reg(current,i,rs2[i]);
1667   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1668     assert(0);
1669   }
1670   #if defined(HOST_IMM8)
1671   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1672   else alloc_reg(current,i,INVCP);
1673   #endif
1674   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) { // SWL/SWL/SDL/SDR
1675     alloc_reg(current,i,FTEMP);
1676   }
1677   // We need a temporary register for address generation
1678   alloc_reg_temp(current,i,-1);
1679   minimum_free_regs[i]=1;
1680 }
1681
1682 void c1ls_alloc(struct regstat *current,int i)
1683 {
1684   //clear_const(current,rs1[i]); // FIXME
1685   clear_const(current,rt1[i]);
1686   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1687   alloc_reg(current,i,CSREG); // Status
1688   alloc_reg(current,i,FTEMP);
1689   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1690     assert(0);
1691   }
1692   #if defined(HOST_IMM8)
1693   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1694   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1695     alloc_reg(current,i,INVCP);
1696   #endif
1697   // We need a temporary register for address generation
1698   alloc_reg_temp(current,i,-1);
1699 }
1700
1701 void c2ls_alloc(struct regstat *current,int i)
1702 {
1703   clear_const(current,rt1[i]);
1704   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1705   alloc_reg(current,i,FTEMP);
1706   #if defined(HOST_IMM8)
1707   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1708   if((opcode[i]&0x3b)==0x3a) // SWC2/SDC2
1709     alloc_reg(current,i,INVCP);
1710   #endif
1711   // We need a temporary register for address generation
1712   alloc_reg_temp(current,i,-1);
1713   minimum_free_regs[i]=1;
1714 }
1715
1716 #ifndef multdiv_alloc
1717 void multdiv_alloc(struct regstat *current,int i)
1718 {
1719   //  case 0x18: MULT
1720   //  case 0x19: MULTU
1721   //  case 0x1A: DIV
1722   //  case 0x1B: DIVU
1723   //  case 0x1C: DMULT
1724   //  case 0x1D: DMULTU
1725   //  case 0x1E: DDIV
1726   //  case 0x1F: DDIVU
1727   clear_const(current,rs1[i]);
1728   clear_const(current,rs2[i]);
1729   if(rs1[i]&&rs2[i])
1730   {
1731     if((opcode2[i]&4)==0) // 32-bit
1732     {
1733       current->u&=~(1LL<<HIREG);
1734       current->u&=~(1LL<<LOREG);
1735       alloc_reg(current,i,HIREG);
1736       alloc_reg(current,i,LOREG);
1737       alloc_reg(current,i,rs1[i]);
1738       alloc_reg(current,i,rs2[i]);
1739       dirty_reg(current,HIREG);
1740       dirty_reg(current,LOREG);
1741     }
1742     else // 64-bit
1743     {
1744       assert(0);
1745     }
1746   }
1747   else
1748   {
1749     // Multiply by zero is zero.
1750     // MIPS does not have a divide by zero exception.
1751     // The result is undefined, we return zero.
1752     alloc_reg(current,i,HIREG);
1753     alloc_reg(current,i,LOREG);
1754     dirty_reg(current,HIREG);
1755     dirty_reg(current,LOREG);
1756   }
1757 }
1758 #endif
1759
1760 void cop0_alloc(struct regstat *current,int i)
1761 {
1762   if(opcode2[i]==0) // MFC0
1763   {
1764     if(rt1[i]) {
1765       clear_const(current,rt1[i]);
1766       alloc_all(current,i);
1767       alloc_reg(current,i,rt1[i]);
1768       dirty_reg(current,rt1[i]);
1769     }
1770   }
1771   else if(opcode2[i]==4) // MTC0
1772   {
1773     if(rs1[i]){
1774       clear_const(current,rs1[i]);
1775       alloc_reg(current,i,rs1[i]);
1776       alloc_all(current,i);
1777     }
1778     else {
1779       alloc_all(current,i); // FIXME: Keep r0
1780       current->u&=~1LL;
1781       alloc_reg(current,i,0);
1782     }
1783   }
1784   else
1785   {
1786     // TLBR/TLBWI/TLBWR/TLBP/ERET
1787     assert(opcode2[i]==0x10);
1788     alloc_all(current,i);
1789   }
1790   minimum_free_regs[i]=HOST_REGS;
1791 }
1792
1793 static void cop12_alloc(struct regstat *current,int i)
1794 {
1795   alloc_reg(current,i,CSREG); // Load status
1796   if(opcode2[i]<3) // MFC1/CFC1
1797   {
1798     if(rt1[i]){
1799       clear_const(current,rt1[i]);
1800       alloc_reg(current,i,rt1[i]);
1801       dirty_reg(current,rt1[i]);
1802     }
1803     alloc_reg_temp(current,i,-1);
1804   }
1805   else if(opcode2[i]>3) // MTC1/CTC1
1806   {
1807     if(rs1[i]){
1808       clear_const(current,rs1[i]);
1809       alloc_reg(current,i,rs1[i]);
1810     }
1811     else {
1812       current->u&=~1LL;
1813       alloc_reg(current,i,0);
1814     }
1815     alloc_reg_temp(current,i,-1);
1816   }
1817   minimum_free_regs[i]=1;
1818 }
1819
1820 void c2op_alloc(struct regstat *current,int i)
1821 {
1822   alloc_reg_temp(current,i,-1);
1823 }
1824
1825 void syscall_alloc(struct regstat *current,int i)
1826 {
1827   alloc_cc(current,i);
1828   dirty_reg(current,CCREG);
1829   alloc_all(current,i);
1830   minimum_free_regs[i]=HOST_REGS;
1831   current->isconst=0;
1832 }
1833
1834 void delayslot_alloc(struct regstat *current,int i)
1835 {
1836   switch(itype[i]) {
1837     case UJUMP:
1838     case CJUMP:
1839     case SJUMP:
1840     case RJUMP:
1841     case SYSCALL:
1842     case HLECALL:
1843     case SPAN:
1844       assem_debug("jump in the delay slot.  this shouldn't happen.\n");//exit(1);
1845       SysPrintf("Disabled speculative precompilation\n");
1846       stop_after_jal=1;
1847       break;
1848     case IMM16:
1849       imm16_alloc(current,i);
1850       break;
1851     case LOAD:
1852     case LOADLR:
1853       load_alloc(current,i);
1854       break;
1855     case STORE:
1856     case STORELR:
1857       store_alloc(current,i);
1858       break;
1859     case ALU:
1860       alu_alloc(current,i);
1861       break;
1862     case SHIFT:
1863       shift_alloc(current,i);
1864       break;
1865     case MULTDIV:
1866       multdiv_alloc(current,i);
1867       break;
1868     case SHIFTIMM:
1869       shiftimm_alloc(current,i);
1870       break;
1871     case MOV:
1872       mov_alloc(current,i);
1873       break;
1874     case COP0:
1875       cop0_alloc(current,i);
1876       break;
1877     case COP1:
1878     case COP2:
1879       cop12_alloc(current,i);
1880       break;
1881     case C1LS:
1882       c1ls_alloc(current,i);
1883       break;
1884     case C2LS:
1885       c2ls_alloc(current,i);
1886       break;
1887     case C2OP:
1888       c2op_alloc(current,i);
1889       break;
1890   }
1891 }
1892
1893 // Special case where a branch and delay slot span two pages in virtual memory
1894 static void pagespan_alloc(struct regstat *current,int i)
1895 {
1896   current->isconst=0;
1897   current->wasconst=0;
1898   regs[i].wasconst=0;
1899   minimum_free_regs[i]=HOST_REGS;
1900   alloc_all(current,i);
1901   alloc_cc(current,i);
1902   dirty_reg(current,CCREG);
1903   if(opcode[i]==3) // JAL
1904   {
1905     alloc_reg(current,i,31);
1906     dirty_reg(current,31);
1907   }
1908   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
1909   {
1910     alloc_reg(current,i,rs1[i]);
1911     if (rt1[i]!=0) {
1912       alloc_reg(current,i,rt1[i]);
1913       dirty_reg(current,rt1[i]);
1914     }
1915   }
1916   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
1917   {
1918     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1919     if(rs2[i]) alloc_reg(current,i,rs2[i]);
1920   }
1921   else
1922   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
1923   {
1924     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1925   }
1926   //else ...
1927 }
1928
1929 static void add_stub(enum stub_type type, void *addr, void *retaddr,
1930   u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e)
1931 {
1932   assert(a < ARRAY_SIZE(stubs));
1933   stubs[stubcount].type = type;
1934   stubs[stubcount].addr = addr;
1935   stubs[stubcount].retaddr = retaddr;
1936   stubs[stubcount].a = a;
1937   stubs[stubcount].b = b;
1938   stubs[stubcount].c = c;
1939   stubs[stubcount].d = d;
1940   stubs[stubcount].e = e;
1941   stubcount++;
1942 }
1943
1944 static void add_stub_r(enum stub_type type, void *addr, void *retaddr,
1945   int i, int addr_reg, struct regstat *i_regs, int ccadj, u_int reglist)
1946 {
1947   add_stub(type, addr, retaddr, i, addr_reg, (uintptr_t)i_regs, ccadj, reglist);
1948 }
1949
1950 // Write out a single register
1951 static void wb_register(signed char r,signed char regmap[],uint64_t dirty)
1952 {
1953   int hr;
1954   for(hr=0;hr<HOST_REGS;hr++) {
1955     if(hr!=EXCLUDE_REG) {
1956       if((regmap[hr]&63)==r) {
1957         if((dirty>>hr)&1) {
1958           assert(regmap[hr]<64);
1959           emit_storereg(r,hr);
1960         }
1961       }
1962     }
1963   }
1964 }
1965
1966 static void wb_valid(signed char pre[],signed char entry[],u_int dirty_pre,u_int dirty,uint64_t u)
1967 {
1968   //if(dirty_pre==dirty) return;
1969   int hr,reg;
1970   for(hr=0;hr<HOST_REGS;hr++) {
1971     if(hr!=EXCLUDE_REG) {
1972       reg=pre[hr];
1973       if(((~u)>>(reg&63))&1) {
1974         if(reg>0) {
1975           if(((dirty_pre&~dirty)>>hr)&1) {
1976             if(reg>0&&reg<34) {
1977               emit_storereg(reg,hr);
1978             }
1979             else if(reg>=64) {
1980               assert(0);
1981             }
1982           }
1983         }
1984       }
1985     }
1986   }
1987 }
1988
1989 void rlist()
1990 {
1991   int i;
1992   printf("TRACE: ");
1993   for(i=0;i<32;i++)
1994     printf("r%d:%8x%8x ",i,((int *)(reg+i))[1],((int *)(reg+i))[0]);
1995   printf("\n");
1996 }
1997
1998 void alu_assemble(int i,struct regstat *i_regs)
1999 {
2000   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2001     if(rt1[i]) {
2002       signed char s1,s2,t;
2003       t=get_reg(i_regs->regmap,rt1[i]);
2004       if(t>=0) {
2005         s1=get_reg(i_regs->regmap,rs1[i]);
2006         s2=get_reg(i_regs->regmap,rs2[i]);
2007         if(rs1[i]&&rs2[i]) {
2008           assert(s1>=0);
2009           assert(s2>=0);
2010           if(opcode2[i]&2) emit_sub(s1,s2,t);
2011           else emit_add(s1,s2,t);
2012         }
2013         else if(rs1[i]) {
2014           if(s1>=0) emit_mov(s1,t);
2015           else emit_loadreg(rs1[i],t);
2016         }
2017         else if(rs2[i]) {
2018           if(s2>=0) {
2019             if(opcode2[i]&2) emit_neg(s2,t);
2020             else emit_mov(s2,t);
2021           }
2022           else {
2023             emit_loadreg(rs2[i],t);
2024             if(opcode2[i]&2) emit_neg(t,t);
2025           }
2026         }
2027         else emit_zeroreg(t);
2028       }
2029     }
2030   }
2031   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2032     assert(0);
2033   }
2034   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2035     if(rt1[i]) {
2036       signed char s1l,s2l,t;
2037       {
2038         t=get_reg(i_regs->regmap,rt1[i]);
2039         //assert(t>=0);
2040         if(t>=0) {
2041           s1l=get_reg(i_regs->regmap,rs1[i]);
2042           s2l=get_reg(i_regs->regmap,rs2[i]);
2043           if(rs2[i]==0) // rx<r0
2044           {
2045             assert(s1l>=0);
2046             if(opcode2[i]==0x2a) // SLT
2047               emit_shrimm(s1l,31,t);
2048             else // SLTU (unsigned can not be less than zero)
2049               emit_zeroreg(t);
2050           }
2051           else if(rs1[i]==0) // r0<rx
2052           {
2053             assert(s2l>=0);
2054             if(opcode2[i]==0x2a) // SLT
2055               emit_set_gz32(s2l,t);
2056             else // SLTU (set if not zero)
2057               emit_set_nz32(s2l,t);
2058           }
2059           else{
2060             assert(s1l>=0);assert(s2l>=0);
2061             if(opcode2[i]==0x2a) // SLT
2062               emit_set_if_less32(s1l,s2l,t);
2063             else // SLTU
2064               emit_set_if_carry32(s1l,s2l,t);
2065           }
2066         }
2067       }
2068     }
2069   }
2070   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2071     if(rt1[i]) {
2072       signed char s1l,s2l,tl;
2073       tl=get_reg(i_regs->regmap,rt1[i]);
2074       {
2075         if(tl>=0) {
2076           s1l=get_reg(i_regs->regmap,rs1[i]);
2077           s2l=get_reg(i_regs->regmap,rs2[i]);
2078           if(rs1[i]&&rs2[i]) {
2079             assert(s1l>=0);
2080             assert(s2l>=0);
2081             if(opcode2[i]==0x24) { // AND
2082               emit_and(s1l,s2l,tl);
2083             } else
2084             if(opcode2[i]==0x25) { // OR
2085               emit_or(s1l,s2l,tl);
2086             } else
2087             if(opcode2[i]==0x26) { // XOR
2088               emit_xor(s1l,s2l,tl);
2089             } else
2090             if(opcode2[i]==0x27) { // NOR
2091               emit_or(s1l,s2l,tl);
2092               emit_not(tl,tl);
2093             }
2094           }
2095           else
2096           {
2097             if(opcode2[i]==0x24) { // AND
2098               emit_zeroreg(tl);
2099             } else
2100             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2101               if(rs1[i]){
2102                 if(s1l>=0) emit_mov(s1l,tl);
2103                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2104               }
2105               else
2106               if(rs2[i]){
2107                 if(s2l>=0) emit_mov(s2l,tl);
2108                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2109               }
2110               else emit_zeroreg(tl);
2111             } else
2112             if(opcode2[i]==0x27) { // NOR
2113               if(rs1[i]){
2114                 if(s1l>=0) emit_not(s1l,tl);
2115                 else {
2116                   emit_loadreg(rs1[i],tl);
2117                   emit_not(tl,tl);
2118                 }
2119               }
2120               else
2121               if(rs2[i]){
2122                 if(s2l>=0) emit_not(s2l,tl);
2123                 else {
2124                   emit_loadreg(rs2[i],tl);
2125                   emit_not(tl,tl);
2126                 }
2127               }
2128               else emit_movimm(-1,tl);
2129             }
2130           }
2131         }
2132       }
2133     }
2134   }
2135 }
2136
2137 void imm16_assemble(int i,struct regstat *i_regs)
2138 {
2139   if (opcode[i]==0x0f) { // LUI
2140     if(rt1[i]) {
2141       signed char t;
2142       t=get_reg(i_regs->regmap,rt1[i]);
2143       //assert(t>=0);
2144       if(t>=0) {
2145         if(!((i_regs->isconst>>t)&1))
2146           emit_movimm(imm[i]<<16,t);
2147       }
2148     }
2149   }
2150   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2151     if(rt1[i]) {
2152       signed char s,t;
2153       t=get_reg(i_regs->regmap,rt1[i]);
2154       s=get_reg(i_regs->regmap,rs1[i]);
2155       if(rs1[i]) {
2156         //assert(t>=0);
2157         //assert(s>=0);
2158         if(t>=0) {
2159           if(!((i_regs->isconst>>t)&1)) {
2160             if(s<0) {
2161               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2162               emit_addimm(t,imm[i],t);
2163             }else{
2164               if(!((i_regs->wasconst>>s)&1))
2165                 emit_addimm(s,imm[i],t);
2166               else
2167                 emit_movimm(constmap[i][s]+imm[i],t);
2168             }
2169           }
2170         }
2171       } else {
2172         if(t>=0) {
2173           if(!((i_regs->isconst>>t)&1))
2174             emit_movimm(imm[i],t);
2175         }
2176       }
2177     }
2178   }
2179   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2180     if(rt1[i]) {
2181       signed char sh,sl,th,tl;
2182       th=get_reg(i_regs->regmap,rt1[i]|64);
2183       tl=get_reg(i_regs->regmap,rt1[i]);
2184       sh=get_reg(i_regs->regmap,rs1[i]|64);
2185       sl=get_reg(i_regs->regmap,rs1[i]);
2186       if(tl>=0) {
2187         if(rs1[i]) {
2188           assert(sh>=0);
2189           assert(sl>=0);
2190           if(th>=0) {
2191             emit_addimm64_32(sh,sl,imm[i],th,tl);
2192           }
2193           else {
2194             emit_addimm(sl,imm[i],tl);
2195           }
2196         } else {
2197           emit_movimm(imm[i],tl);
2198           if(th>=0) emit_movimm(((signed int)imm[i])>>31,th);
2199         }
2200       }
2201     }
2202   }
2203   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2204     if(rt1[i]) {
2205       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2206       signed char sl,t;
2207       t=get_reg(i_regs->regmap,rt1[i]);
2208       sl=get_reg(i_regs->regmap,rs1[i]);
2209       //assert(t>=0);
2210       if(t>=0) {
2211         if(rs1[i]>0) {
2212             if(opcode[i]==0x0a) { // SLTI
2213               if(sl<0) {
2214                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2215                 emit_slti32(t,imm[i],t);
2216               }else{
2217                 emit_slti32(sl,imm[i],t);
2218               }
2219             }
2220             else { // SLTIU
2221               if(sl<0) {
2222                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2223                 emit_sltiu32(t,imm[i],t);
2224               }else{
2225                 emit_sltiu32(sl,imm[i],t);
2226               }
2227             }
2228         }else{
2229           // SLTI(U) with r0 is just stupid,
2230           // nonetheless examples can be found
2231           if(opcode[i]==0x0a) // SLTI
2232             if(0<imm[i]) emit_movimm(1,t);
2233             else emit_zeroreg(t);
2234           else // SLTIU
2235           {
2236             if(imm[i]) emit_movimm(1,t);
2237             else emit_zeroreg(t);
2238           }
2239         }
2240       }
2241     }
2242   }
2243   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2244     if(rt1[i]) {
2245       signed char sh,sl,th,tl;
2246       th=get_reg(i_regs->regmap,rt1[i]|64);
2247       tl=get_reg(i_regs->regmap,rt1[i]);
2248       sh=get_reg(i_regs->regmap,rs1[i]|64);
2249       sl=get_reg(i_regs->regmap,rs1[i]);
2250       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2251         if(opcode[i]==0x0c) //ANDI
2252         {
2253           if(rs1[i]) {
2254             if(sl<0) {
2255               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2256               emit_andimm(tl,imm[i],tl);
2257             }else{
2258               if(!((i_regs->wasconst>>sl)&1))
2259                 emit_andimm(sl,imm[i],tl);
2260               else
2261                 emit_movimm(constmap[i][sl]&imm[i],tl);
2262             }
2263           }
2264           else
2265             emit_zeroreg(tl);
2266           if(th>=0) emit_zeroreg(th);
2267         }
2268         else
2269         {
2270           if(rs1[i]) {
2271             if(sl<0) {
2272               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2273             }
2274             if(th>=0) {
2275               if(sh<0) {
2276                 emit_loadreg(rs1[i]|64,th);
2277               }else{
2278                 emit_mov(sh,th);
2279               }
2280             }
2281             if(opcode[i]==0x0d) { // ORI
2282               if(sl<0) {
2283                 emit_orimm(tl,imm[i],tl);
2284               }else{
2285                 if(!((i_regs->wasconst>>sl)&1))
2286                   emit_orimm(sl,imm[i],tl);
2287                 else
2288                   emit_movimm(constmap[i][sl]|imm[i],tl);
2289               }
2290             }
2291             if(opcode[i]==0x0e) { // XORI
2292               if(sl<0) {
2293                 emit_xorimm(tl,imm[i],tl);
2294               }else{
2295                 if(!((i_regs->wasconst>>sl)&1))
2296                   emit_xorimm(sl,imm[i],tl);
2297                 else
2298                   emit_movimm(constmap[i][sl]^imm[i],tl);
2299               }
2300             }
2301           }
2302           else {
2303             emit_movimm(imm[i],tl);
2304             if(th>=0) emit_zeroreg(th);
2305           }
2306         }
2307       }
2308     }
2309   }
2310 }
2311
2312 void shiftimm_assemble(int i,struct regstat *i_regs)
2313 {
2314   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2315   {
2316     if(rt1[i]) {
2317       signed char s,t;
2318       t=get_reg(i_regs->regmap,rt1[i]);
2319       s=get_reg(i_regs->regmap,rs1[i]);
2320       //assert(t>=0);
2321       if(t>=0&&!((i_regs->isconst>>t)&1)){
2322         if(rs1[i]==0)
2323         {
2324           emit_zeroreg(t);
2325         }
2326         else
2327         {
2328           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2329           if(imm[i]) {
2330             if(opcode2[i]==0) // SLL
2331             {
2332               emit_shlimm(s<0?t:s,imm[i],t);
2333             }
2334             if(opcode2[i]==2) // SRL
2335             {
2336               emit_shrimm(s<0?t:s,imm[i],t);
2337             }
2338             if(opcode2[i]==3) // SRA
2339             {
2340               emit_sarimm(s<0?t:s,imm[i],t);
2341             }
2342           }else{
2343             // Shift by zero
2344             if(s>=0 && s!=t) emit_mov(s,t);
2345           }
2346         }
2347       }
2348       //emit_storereg(rt1[i],t); //DEBUG
2349     }
2350   }
2351   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2352   {
2353     assert(0);
2354   }
2355   if(opcode2[i]==0x3c) // DSLL32
2356   {
2357     assert(0);
2358   }
2359   if(opcode2[i]==0x3e) // DSRL32
2360   {
2361     assert(0);
2362   }
2363   if(opcode2[i]==0x3f) // DSRA32
2364   {
2365     assert(0);
2366   }
2367 }
2368
2369 #ifndef shift_assemble
2370 void shift_assemble(int i,struct regstat *i_regs)
2371 {
2372   printf("Need shift_assemble for this architecture.\n");
2373   exit(1);
2374 }
2375 #endif
2376
2377 enum {
2378   MTYPE_8000 = 0,
2379   MTYPE_8020,
2380   MTYPE_0000,
2381   MTYPE_A000,
2382   MTYPE_1F80,
2383 };
2384
2385 static int get_ptr_mem_type(u_int a)
2386 {
2387   if(a < 0x00200000) {
2388     if(a<0x1000&&((start>>20)==0xbfc||(start>>24)==0xa0))
2389       // return wrong, must use memhandler for BIOS self-test to pass
2390       // 007 does similar stuff from a00 mirror, weird stuff
2391       return MTYPE_8000;
2392     return MTYPE_0000;
2393   }
2394   if(0x1f800000 <= a && a < 0x1f801000)
2395     return MTYPE_1F80;
2396   if(0x80200000 <= a && a < 0x80800000)
2397     return MTYPE_8020;
2398   if(0xa0000000 <= a && a < 0xa0200000)
2399     return MTYPE_A000;
2400   return MTYPE_8000;
2401 }
2402
2403 static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override)
2404 {
2405   void *jaddr = NULL;
2406   int type=0;
2407   int mr=rs1[i];
2408   if(((smrv_strong|smrv_weak)>>mr)&1) {
2409     type=get_ptr_mem_type(smrv[mr]);
2410     //printf("set %08x @%08x r%d %d\n", smrv[mr], start+i*4, mr, type);
2411   }
2412   else {
2413     // use the mirror we are running on
2414     type=get_ptr_mem_type(start);
2415     //printf("set nospec   @%08x r%d %d\n", start+i*4, mr, type);
2416   }
2417
2418   if(type==MTYPE_8020) { // RAM 80200000+ mirror
2419     emit_andimm(addr,~0x00e00000,HOST_TEMPREG);
2420     addr=*addr_reg_override=HOST_TEMPREG;
2421     type=0;
2422   }
2423   else if(type==MTYPE_0000) { // RAM 0 mirror
2424     emit_orimm(addr,0x80000000,HOST_TEMPREG);
2425     addr=*addr_reg_override=HOST_TEMPREG;
2426     type=0;
2427   }
2428   else if(type==MTYPE_A000) { // RAM A mirror
2429     emit_andimm(addr,~0x20000000,HOST_TEMPREG);
2430     addr=*addr_reg_override=HOST_TEMPREG;
2431     type=0;
2432   }
2433   else if(type==MTYPE_1F80) { // scratchpad
2434     if (psxH == (void *)0x1f800000) {
2435       emit_addimm(addr,-0x1f800000,HOST_TEMPREG);
2436       emit_cmpimm(HOST_TEMPREG,0x1000);
2437       jaddr=out;
2438       emit_jc(0);
2439     }
2440     else {
2441       // do the usual RAM check, jump will go to the right handler
2442       type=0;
2443     }
2444   }
2445
2446   if(type==0)
2447   {
2448     emit_cmpimm(addr,RAM_SIZE);
2449     jaddr=out;
2450     #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2451     // Hint to branch predictor that the branch is unlikely to be taken
2452     if(rs1[i]>=28)
2453       emit_jno_unlikely(0);
2454     else
2455     #endif
2456       emit_jno(0);
2457     if(ram_offset!=0) {
2458       emit_addimm(addr,ram_offset,HOST_TEMPREG);
2459       addr=*addr_reg_override=HOST_TEMPREG;
2460     }
2461   }
2462
2463   return jaddr;
2464 }
2465
2466 static void load_assemble(int i,struct regstat *i_regs)
2467 {
2468   int s,th,tl,addr;
2469   int offset;
2470   void *jaddr=0;
2471   int memtarget=0,c=0;
2472   int fastload_reg_override=0;
2473   u_int hr,reglist=0;
2474   th=get_reg(i_regs->regmap,rt1[i]|64);
2475   tl=get_reg(i_regs->regmap,rt1[i]);
2476   s=get_reg(i_regs->regmap,rs1[i]);
2477   offset=imm[i];
2478   for(hr=0;hr<HOST_REGS;hr++) {
2479     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2480   }
2481   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2482   if(s>=0) {
2483     c=(i_regs->wasconst>>s)&1;
2484     if (c) {
2485       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2486     }
2487   }
2488   //printf("load_assemble: c=%d\n",c);
2489   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2490   // FIXME: Even if the load is a NOP, we should check for pagefaults...
2491   if((tl<0&&(!c||(((u_int)constmap[i][s]+offset)>>16)==0x1f80))
2492     ||rt1[i]==0) {
2493       // could be FIFO, must perform the read
2494       // ||dummy read
2495       assem_debug("(forced read)\n");
2496       tl=get_reg(i_regs->regmap,-1);
2497       assert(tl>=0);
2498   }
2499   if(offset||s<0||c) addr=tl;
2500   else addr=s;
2501   //if(tl<0) tl=get_reg(i_regs->regmap,-1);
2502  if(tl>=0) {
2503   //printf("load_assemble: c=%d\n",c);
2504   //if(c) printf("load_assemble: const=%lx\n",(long)constmap[i][s]+offset);
2505   assert(tl>=0); // Even if the load is a NOP, we must check for pagefaults and I/O
2506   reglist&=~(1<<tl);
2507   if(th>=0) reglist&=~(1<<th);
2508   if(!c) {
2509     #ifdef R29_HACK
2510     // Strmnnrmn's speed hack
2511     if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
2512     #endif
2513     {
2514       jaddr=emit_fastpath_cmp_jump(i,addr,&fastload_reg_override);
2515     }
2516   }
2517   else if(ram_offset&&memtarget) {
2518     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2519     fastload_reg_override=HOST_TEMPREG;
2520   }
2521   int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg
2522   if (opcode[i]==0x20) { // LB
2523     if(!c||memtarget) {
2524       if(!dummy) {
2525         {
2526           int x=0,a=tl;
2527           if(!c) a=addr;
2528           if(fastload_reg_override) a=fastload_reg_override;
2529
2530           emit_movsbl_indexed(x,a,tl);
2531         }
2532       }
2533       if(jaddr)
2534         add_stub_r(LOADB_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2535     }
2536     else
2537       inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2538   }
2539   if (opcode[i]==0x21) { // LH
2540     if(!c||memtarget) {
2541       if(!dummy) {
2542         int x=0,a=tl;
2543         if(!c) a=addr;
2544         if(fastload_reg_override) a=fastload_reg_override;
2545         emit_movswl_indexed(x,a,tl);
2546       }
2547       if(jaddr)
2548         add_stub_r(LOADH_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2549     }
2550     else
2551       inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2552   }
2553   if (opcode[i]==0x23) { // LW
2554     if(!c||memtarget) {
2555       if(!dummy) {
2556         int a=addr;
2557         if(fastload_reg_override) a=fastload_reg_override;
2558         emit_readword_indexed(0,a,tl);
2559       }
2560       if(jaddr)
2561         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2562     }
2563     else
2564       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2565   }
2566   if (opcode[i]==0x24) { // LBU
2567     if(!c||memtarget) {
2568       if(!dummy) {
2569         int x=0,a=tl;
2570         if(!c) a=addr;
2571         if(fastload_reg_override) a=fastload_reg_override;
2572
2573         emit_movzbl_indexed(x,a,tl);
2574       }
2575       if(jaddr)
2576         add_stub_r(LOADBU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2577     }
2578     else
2579       inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2580   }
2581   if (opcode[i]==0x25) { // LHU
2582     if(!c||memtarget) {
2583       if(!dummy) {
2584         int x=0,a=tl;
2585         if(!c) a=addr;
2586         if(fastload_reg_override) a=fastload_reg_override;
2587         emit_movzwl_indexed(x,a,tl);
2588       }
2589       if(jaddr)
2590         add_stub_r(LOADHU_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2591     }
2592     else
2593       inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2594   }
2595   if (opcode[i]==0x27) { // LWU
2596     assert(th>=0);
2597     if(!c||memtarget) {
2598       if(!dummy) {
2599         int a=addr;
2600         if(fastload_reg_override) a=fastload_reg_override;
2601         emit_readword_indexed(0,a,tl);
2602       }
2603       if(jaddr)
2604         add_stub_r(LOADW_STUB,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2605     }
2606     else {
2607       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2608     }
2609     emit_zeroreg(th);
2610   }
2611   if (opcode[i]==0x37) { // LD
2612     assert(0);
2613   }
2614  }
2615 }
2616
2617 #ifndef loadlr_assemble
2618 void loadlr_assemble(int i,struct regstat *i_regs)
2619 {
2620   printf("Need loadlr_assemble for this architecture.\n");
2621   exit(1);
2622 }
2623 #endif
2624
2625 void store_assemble(int i,struct regstat *i_regs)
2626 {
2627   int s,tl;
2628   int addr,temp;
2629   int offset;
2630   void *jaddr=0;
2631   enum stub_type type;
2632   int memtarget=0,c=0;
2633   int agr=AGEN1+(i&1);
2634   int faststore_reg_override=0;
2635   u_int hr,reglist=0;
2636   tl=get_reg(i_regs->regmap,rs2[i]);
2637   s=get_reg(i_regs->regmap,rs1[i]);
2638   temp=get_reg(i_regs->regmap,agr);
2639   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2640   offset=imm[i];
2641   if(s>=0) {
2642     c=(i_regs->wasconst>>s)&1;
2643     if(c) {
2644       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2645     }
2646   }
2647   assert(tl>=0);
2648   assert(temp>=0);
2649   for(hr=0;hr<HOST_REGS;hr++) {
2650     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2651   }
2652   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2653   if(offset||s<0||c) addr=temp;
2654   else addr=s;
2655   if(!c) {
2656     jaddr=emit_fastpath_cmp_jump(i,addr,&faststore_reg_override);
2657   }
2658   else if(ram_offset&&memtarget) {
2659     emit_addimm(addr,ram_offset,HOST_TEMPREG);
2660     faststore_reg_override=HOST_TEMPREG;
2661   }
2662
2663   if (opcode[i]==0x28) { // SB
2664     if(!c||memtarget) {
2665       int x=0,a=temp;
2666       if(!c) a=addr;
2667       if(faststore_reg_override) a=faststore_reg_override;
2668       emit_writebyte_indexed(tl,x,a);
2669     }
2670     type=STOREB_STUB;
2671   }
2672   if (opcode[i]==0x29) { // SH
2673     if(!c||memtarget) {
2674       int x=0,a=temp;
2675       if(!c) a=addr;
2676       if(faststore_reg_override) a=faststore_reg_override;
2677       emit_writehword_indexed(tl,x,a);
2678     }
2679     type=STOREH_STUB;
2680   }
2681   if (opcode[i]==0x2B) { // SW
2682     if(!c||memtarget) {
2683       int a=addr;
2684       if(faststore_reg_override) a=faststore_reg_override;
2685       emit_writeword_indexed(tl,0,a);
2686     }
2687     type=STOREW_STUB;
2688   }
2689   if (opcode[i]==0x3F) { // SD
2690     assert(0);
2691     type=STORED_STUB;
2692   }
2693   if(jaddr) {
2694     // PCSX store handlers don't check invcode again
2695     reglist|=1<<addr;
2696     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2697     jaddr=0;
2698   }
2699   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
2700     if(!c||memtarget) {
2701       #ifdef DESTRUCTIVE_SHIFT
2702       // The x86 shift operation is 'destructive'; it overwrites the
2703       // source register, so we need to make a copy first and use that.
2704       addr=temp;
2705       #endif
2706       #if defined(HOST_IMM8)
2707       int ir=get_reg(i_regs->regmap,INVCP);
2708       assert(ir>=0);
2709       emit_cmpmem_indexedsr12_reg(ir,addr,1);
2710       #else
2711       emit_cmpmem_indexedsr12_imm(invalid_code,addr,1);
2712       #endif
2713       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2714       emit_callne(invalidate_addr_reg[addr]);
2715       #else
2716       void *jaddr2 = out;
2717       emit_jne(0);
2718       add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),addr,0,0,0);
2719       #endif
2720     }
2721   }
2722   u_int addr_val=constmap[i][s]+offset;
2723   if(jaddr) {
2724     add_stub_r(type,jaddr,out,i,addr,i_regs,ccadj[i],reglist);
2725   } else if(c&&!memtarget) {
2726     inline_writestub(type,i,addr_val,i_regs->regmap,rs2[i],ccadj[i],reglist);
2727   }
2728   // basic current block modification detection..
2729   // not looking back as that should be in mips cache already
2730   if(c&&start+i*4<addr_val&&addr_val<start+slen*4) {
2731     SysPrintf("write to %08x hits block %08x, pc=%08x\n",addr_val,start,start+i*4);
2732     assert(i_regs->regmap==regs[i].regmap); // not delay slot
2733     if(i_regs->regmap==regs[i].regmap) {
2734       load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
2735       wb_dirtys(regs[i].regmap_entry,regs[i].wasdirty);
2736       emit_movimm(start+i*4+4,0);
2737       emit_writeword(0,&pcaddr);
2738       emit_jmp(do_interrupt);
2739     }
2740   }
2741 }
2742
2743 void storelr_assemble(int i,struct regstat *i_regs)
2744 {
2745   int s,tl;
2746   int temp;
2747   int offset;
2748   void *jaddr=0;
2749   void *case1, *case2, *case3;
2750   void *done0, *done1, *done2;
2751   int memtarget=0,c=0;
2752   int agr=AGEN1+(i&1);
2753   u_int hr,reglist=0;
2754   tl=get_reg(i_regs->regmap,rs2[i]);
2755   s=get_reg(i_regs->regmap,rs1[i]);
2756   temp=get_reg(i_regs->regmap,agr);
2757   if(temp<0) temp=get_reg(i_regs->regmap,-1);
2758   offset=imm[i];
2759   if(s>=0) {
2760     c=(i_regs->isconst>>s)&1;
2761     if(c) {
2762       memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2763     }
2764   }
2765   assert(tl>=0);
2766   for(hr=0;hr<HOST_REGS;hr++) {
2767     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2768   }
2769   assert(temp>=0);
2770   if(!c) {
2771     emit_cmpimm(s<0||offset?temp:s,RAM_SIZE);
2772     if(!offset&&s!=temp) emit_mov(s,temp);
2773     jaddr=out;
2774     emit_jno(0);
2775   }
2776   else
2777   {
2778     if(!memtarget||!rs1[i]) {
2779       jaddr=out;
2780       emit_jmp(0);
2781     }
2782   }
2783   emit_addimm_no_flags(ram_offset,temp);
2784
2785   if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
2786     assert(0);
2787   }
2788
2789   emit_xorimm(temp,3,temp);
2790   emit_testimm(temp,2);
2791   case2=out;
2792   emit_jne(0);
2793   emit_testimm(temp,1);
2794   case1=out;
2795   emit_jne(0);
2796   // 0
2797   if (opcode[i]==0x2A) { // SWL
2798     emit_writeword_indexed(tl,0,temp);
2799   }
2800   if (opcode[i]==0x2E) { // SWR
2801     emit_writebyte_indexed(tl,3,temp);
2802   }
2803   if (opcode[i]==0x2C) { // SDL
2804     assert(0);
2805   }
2806   if (opcode[i]==0x2D) { // SDR
2807     assert(0);
2808   }
2809   done0=out;
2810   emit_jmp(0);
2811   // 1
2812   set_jump_target(case1, out);
2813   if (opcode[i]==0x2A) { // SWL
2814     // Write 3 msb into three least significant bytes
2815     if(rs2[i]) emit_rorimm(tl,8,tl);
2816     emit_writehword_indexed(tl,-1,temp);
2817     if(rs2[i]) emit_rorimm(tl,16,tl);
2818     emit_writebyte_indexed(tl,1,temp);
2819     if(rs2[i]) emit_rorimm(tl,8,tl);
2820   }
2821   if (opcode[i]==0x2E) { // SWR
2822     // Write two lsb into two most significant bytes
2823     emit_writehword_indexed(tl,1,temp);
2824   }
2825   if (opcode[i]==0x2C) { // SDL
2826     assert(0);
2827   }
2828   if (opcode[i]==0x2D) { // SDR
2829     assert(0);
2830   }
2831   done1=out;
2832   emit_jmp(0);
2833   // 2
2834   set_jump_target(case2, out);
2835   emit_testimm(temp,1);
2836   case3=out;
2837   emit_jne(0);
2838   if (opcode[i]==0x2A) { // SWL
2839     // Write two msb into two least significant bytes
2840     if(rs2[i]) emit_rorimm(tl,16,tl);
2841     emit_writehword_indexed(tl,-2,temp);
2842     if(rs2[i]) emit_rorimm(tl,16,tl);
2843   }
2844   if (opcode[i]==0x2E) { // SWR
2845     // Write 3 lsb into three most significant bytes
2846     emit_writebyte_indexed(tl,-1,temp);
2847     if(rs2[i]) emit_rorimm(tl,8,tl);
2848     emit_writehword_indexed(tl,0,temp);
2849     if(rs2[i]) emit_rorimm(tl,24,tl);
2850   }
2851   if (opcode[i]==0x2C) { // SDL
2852     assert(0);
2853   }
2854   if (opcode[i]==0x2D) { // SDR
2855     assert(0);
2856   }
2857   done2=out;
2858   emit_jmp(0);
2859   // 3
2860   set_jump_target(case3, out);
2861   if (opcode[i]==0x2A) { // SWL
2862     // Write msb into least significant byte
2863     if(rs2[i]) emit_rorimm(tl,24,tl);
2864     emit_writebyte_indexed(tl,-3,temp);
2865     if(rs2[i]) emit_rorimm(tl,8,tl);
2866   }
2867   if (opcode[i]==0x2E) { // SWR
2868     // Write entire word
2869     emit_writeword_indexed(tl,-3,temp);
2870   }
2871   if (opcode[i]==0x2C) { // SDL
2872     assert(0);
2873   }
2874   if (opcode[i]==0x2D) { // SDR
2875     assert(0);
2876   }
2877   set_jump_target(done0, out);
2878   set_jump_target(done1, out);
2879   set_jump_target(done2, out);
2880   if (opcode[i]==0x2C) { // SDL
2881     assert(0);
2882   }
2883   if (opcode[i]==0x2D) { // SDR
2884     assert(0);
2885   }
2886   if(!c||!memtarget)
2887     add_stub_r(STORELR_STUB,jaddr,out,i,temp,i_regs,ccadj[i],reglist);
2888   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
2889     emit_addimm_no_flags(-ram_offset,temp);
2890     #if defined(HOST_IMM8)
2891     int ir=get_reg(i_regs->regmap,INVCP);
2892     assert(ir>=0);
2893     emit_cmpmem_indexedsr12_reg(ir,temp,1);
2894     #else
2895     emit_cmpmem_indexedsr12_imm(invalid_code,temp,1);
2896     #endif
2897     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
2898     emit_callne(invalidate_addr_reg[temp]);
2899     #else
2900     void *jaddr2 = out;
2901     emit_jne(0);
2902     add_stub(INVCODE_STUB,jaddr2,out,reglist|(1<<HOST_CCREG),temp,0,0,0);
2903     #endif
2904   }
2905 }
2906
2907 static void cop0_assemble(int i,struct regstat *i_regs)
2908 {
2909   if(opcode2[i]==0) // MFC0
2910   {
2911     signed char t=get_reg(i_regs->regmap,rt1[i]);
2912     u_int copr=(source[i]>>11)&0x1f;
2913     //assert(t>=0); // Why does this happen?  OOT is weird
2914     if(t>=0&&rt1[i]!=0) {
2915       emit_readword(&reg_cop0[copr],t);
2916     }
2917   }
2918   else if(opcode2[i]==4) // MTC0
2919   {
2920     signed char s=get_reg(i_regs->regmap,rs1[i]);
2921     char copr=(source[i]>>11)&0x1f;
2922     assert(s>=0);
2923     wb_register(rs1[i],i_regs->regmap,i_regs->dirty);
2924     if(copr==9||copr==11||copr==12||copr==13) {
2925       emit_readword(&last_count,HOST_TEMPREG);
2926       emit_loadreg(CCREG,HOST_CCREG); // TODO: do proper reg alloc
2927       emit_add(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
2928       emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
2929       emit_writeword(HOST_CCREG,&Count);
2930     }
2931     // What a mess.  The status register (12) can enable interrupts,
2932     // so needs a special case to handle a pending interrupt.
2933     // The interrupt must be taken immediately, because a subsequent
2934     // instruction might disable interrupts again.
2935     if(copr==12||copr==13) {
2936       if (is_delayslot) {
2937         // burn cycles to cause cc_interrupt, which will
2938         // reschedule next_interupt. Relies on CCREG from above.
2939         assem_debug("MTC0 DS %d\n", copr);
2940         emit_writeword(HOST_CCREG,&last_count);
2941         emit_movimm(0,HOST_CCREG);
2942         emit_storereg(CCREG,HOST_CCREG);
2943         emit_loadreg(rs1[i],1);
2944         emit_movimm(copr,0);
2945         emit_call(pcsx_mtc0_ds);
2946         emit_loadreg(rs1[i],s);
2947         return;
2948       }
2949       emit_movimm(start+i*4+4,HOST_TEMPREG);
2950       emit_writeword(HOST_TEMPREG,&pcaddr);
2951       emit_movimm(0,HOST_TEMPREG);
2952       emit_writeword(HOST_TEMPREG,&pending_exception);
2953     }
2954     //else if(copr==12&&is_delayslot) emit_call((int)MTC0_R12);
2955     //else
2956     if(s==HOST_CCREG)
2957       emit_loadreg(rs1[i],1);
2958     else if(s!=1)
2959       emit_mov(s,1);
2960     emit_movimm(copr,0);
2961     emit_call(pcsx_mtc0);
2962     if(copr==9||copr==11||copr==12||copr==13) {
2963       emit_readword(&Count,HOST_CCREG);
2964       emit_readword(&next_interupt,HOST_TEMPREG);
2965       emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
2966       emit_sub(HOST_CCREG,HOST_TEMPREG,HOST_CCREG);
2967       emit_writeword(HOST_TEMPREG,&last_count);
2968       emit_storereg(CCREG,HOST_CCREG);
2969     }
2970     if(copr==12||copr==13) {
2971       assert(!is_delayslot);
2972       emit_readword(&pending_exception,14);
2973       emit_test(14,14);
2974       emit_jne(&do_interrupt);
2975     }
2976     emit_loadreg(rs1[i],s);
2977     if(get_reg(i_regs->regmap,rs1[i]|64)>=0)
2978       emit_loadreg(rs1[i]|64,get_reg(i_regs->regmap,rs1[i]|64));
2979   }
2980   else
2981   {
2982     assert(opcode2[i]==0x10);
2983     //if((source[i]&0x3f)==0x10) // RFE
2984     {
2985       emit_readword(&Status,0);
2986       emit_andimm(0,0x3c,1);
2987       emit_andimm(0,~0xf,0);
2988       emit_orrshr_imm(1,2,0);
2989       emit_writeword(0,&Status);
2990     }
2991   }
2992 }
2993
2994 static void cop1_unusable(int i,struct regstat *i_regs)
2995 {
2996   // XXX: should just just do the exception instead
2997   //if(!cop1_usable)
2998   {
2999     void *jaddr=out;
3000     emit_jmp(0);
3001     add_stub_r(FP_STUB,jaddr,out,i,0,i_regs,is_delayslot,0);
3002   }
3003 }
3004
3005 static void cop1_assemble(int i,struct regstat *i_regs)
3006 {
3007   cop1_unusable(i, i_regs);
3008 }
3009
3010 static void c1ls_assemble(int i,struct regstat *i_regs)
3011 {
3012   cop1_unusable(i, i_regs);
3013 }
3014
3015 // FP_STUB
3016 static void do_cop1stub(int n)
3017 {
3018   literal_pool(256);
3019   assem_debug("do_cop1stub %x\n",start+stubs[n].a*4);
3020   set_jump_target(stubs[n].addr, out);
3021   int i=stubs[n].a;
3022 //  int rs=stubs[n].b;
3023   struct regstat *i_regs=(struct regstat *)stubs[n].c;
3024   int ds=stubs[n].d;
3025   if(!ds) {
3026     load_all_consts(regs[i].regmap_entry,regs[i].wasdirty,i);
3027     //if(i_regs!=&regs[i]) printf("oops: regs[i]=%x i_regs=%x",(int)&regs[i],(int)i_regs);
3028   }
3029   //else {printf("fp exception in delay slot\n");}
3030   wb_dirtys(i_regs->regmap_entry,i_regs->wasdirty);
3031   if(regs[i].regmap_entry[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
3032   emit_movimm(start+(i-ds)*4,EAX); // Get PC
3033   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3034   emit_jmp(ds?fp_exception_ds:fp_exception);
3035 }
3036
3037 static void cop2_get_dreg(u_int copr,signed char tl,signed char temp)
3038 {
3039   switch (copr) {
3040     case 1:
3041     case 3:
3042     case 5:
3043     case 8:
3044     case 9:
3045     case 10:
3046     case 11:
3047       emit_readword(&reg_cop2d[copr],tl);
3048       emit_signextend16(tl,tl);
3049       emit_writeword(tl,&reg_cop2d[copr]); // hmh
3050       break;
3051     case 7:
3052     case 16:
3053     case 17:
3054     case 18:
3055     case 19:
3056       emit_readword(&reg_cop2d[copr],tl);
3057       emit_andimm(tl,0xffff,tl);
3058       emit_writeword(tl,&reg_cop2d[copr]);
3059       break;
3060     case 15:
3061       emit_readword(&reg_cop2d[14],tl); // SXY2
3062       emit_writeword(tl,&reg_cop2d[copr]);
3063       break;
3064     case 28:
3065     case 29:
3066       emit_readword(&reg_cop2d[9],temp);
3067       emit_testimm(temp,0x8000); // do we need this?
3068       emit_andimm(temp,0xf80,temp);
3069       emit_andne_imm(temp,0,temp);
3070       emit_shrimm(temp,7,tl);
3071       emit_readword(&reg_cop2d[10],temp);
3072       emit_testimm(temp,0x8000);
3073       emit_andimm(temp,0xf80,temp);
3074       emit_andne_imm(temp,0,temp);
3075       emit_orrshr_imm(temp,2,tl);
3076       emit_readword(&reg_cop2d[11],temp);
3077       emit_testimm(temp,0x8000);
3078       emit_andimm(temp,0xf80,temp);
3079       emit_andne_imm(temp,0,temp);
3080       emit_orrshl_imm(temp,3,tl);
3081       emit_writeword(tl,&reg_cop2d[copr]);
3082       break;
3083     default:
3084       emit_readword(&reg_cop2d[copr],tl);
3085       break;
3086   }
3087 }
3088
3089 static void cop2_put_dreg(u_int copr,signed char sl,signed char temp)
3090 {
3091   switch (copr) {
3092     case 15:
3093       emit_readword(&reg_cop2d[13],temp);  // SXY1
3094       emit_writeword(sl,&reg_cop2d[copr]);
3095       emit_writeword(temp,&reg_cop2d[12]); // SXY0
3096       emit_readword(&reg_cop2d[14],temp);  // SXY2
3097       emit_writeword(sl,&reg_cop2d[14]);
3098       emit_writeword(temp,&reg_cop2d[13]); // SXY1
3099       break;
3100     case 28:
3101       emit_andimm(sl,0x001f,temp);
3102       emit_shlimm(temp,7,temp);
3103       emit_writeword(temp,&reg_cop2d[9]);
3104       emit_andimm(sl,0x03e0,temp);
3105       emit_shlimm(temp,2,temp);
3106       emit_writeword(temp,&reg_cop2d[10]);
3107       emit_andimm(sl,0x7c00,temp);
3108       emit_shrimm(temp,3,temp);
3109       emit_writeword(temp,&reg_cop2d[11]);
3110       emit_writeword(sl,&reg_cop2d[28]);
3111       break;
3112     case 30:
3113       emit_movs(sl,temp);
3114       emit_mvnmi(temp,temp);
3115 #ifdef HAVE_ARMV5
3116       emit_clz(temp,temp);
3117 #else
3118       emit_movs(temp,HOST_TEMPREG);
3119       emit_movimm(0,temp);
3120       emit_jeq((int)out+4*4);
3121       emit_addpl_imm(temp,1,temp);
3122       emit_lslpls_imm(HOST_TEMPREG,1,HOST_TEMPREG);
3123       emit_jns((int)out-2*4);
3124 #endif
3125       emit_writeword(sl,&reg_cop2d[30]);
3126       emit_writeword(temp,&reg_cop2d[31]);
3127       break;
3128     case 31:
3129       break;
3130     default:
3131       emit_writeword(sl,&reg_cop2d[copr]);
3132       break;
3133   }
3134 }
3135
3136 static void c2ls_assemble(int i,struct regstat *i_regs)
3137 {
3138   int s,tl;
3139   int ar;
3140   int offset;
3141   int memtarget=0,c=0;
3142   void *jaddr2=NULL;
3143   enum stub_type type;
3144   int agr=AGEN1+(i&1);
3145   int fastio_reg_override=0;
3146   u_int hr,reglist=0;
3147   u_int copr=(source[i]>>16)&0x1f;
3148   s=get_reg(i_regs->regmap,rs1[i]);
3149   tl=get_reg(i_regs->regmap,FTEMP);
3150   offset=imm[i];
3151   assert(rs1[i]>0);
3152   assert(tl>=0);
3153
3154   for(hr=0;hr<HOST_REGS;hr++) {
3155     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3156   }
3157   if(i_regs->regmap[HOST_CCREG]==CCREG)
3158     reglist&=~(1<<HOST_CCREG);
3159
3160   // get the address
3161   if (opcode[i]==0x3a) { // SWC2
3162     ar=get_reg(i_regs->regmap,agr);
3163     if(ar<0) ar=get_reg(i_regs->regmap,-1);
3164     reglist|=1<<ar;
3165   } else { // LWC2
3166     ar=tl;
3167   }
3168   if(s>=0) c=(i_regs->wasconst>>s)&1;
3169   memtarget=c&&(((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE);
3170   if (!offset&&!c&&s>=0) ar=s;
3171   assert(ar>=0);
3172
3173   if (opcode[i]==0x3a) { // SWC2
3174     cop2_get_dreg(copr,tl,HOST_TEMPREG);
3175     type=STOREW_STUB;
3176   }
3177   else
3178     type=LOADW_STUB;
3179
3180   if(c&&!memtarget) {
3181     jaddr2=out;
3182     emit_jmp(0); // inline_readstub/inline_writestub?
3183   }
3184   else {
3185     if(!c) {
3186       jaddr2=emit_fastpath_cmp_jump(i,ar,&fastio_reg_override);
3187     }
3188     else if(ram_offset&&memtarget) {
3189       emit_addimm(ar,ram_offset,HOST_TEMPREG);
3190       fastio_reg_override=HOST_TEMPREG;
3191     }
3192     if (opcode[i]==0x32) { // LWC2
3193       int a=ar;
3194       if(fastio_reg_override) a=fastio_reg_override;
3195       emit_readword_indexed(0,a,tl);
3196     }
3197     if (opcode[i]==0x3a) { // SWC2
3198       #ifdef DESTRUCTIVE_SHIFT
3199       if(!offset&&!c&&s>=0) emit_mov(s,ar);
3200       #endif
3201       int a=ar;
3202       if(fastio_reg_override) a=fastio_reg_override;
3203       emit_writeword_indexed(tl,0,a);
3204     }
3205   }
3206   if(jaddr2)
3207     add_stub_r(type,jaddr2,out,i,ar,i_regs,ccadj[i],reglist);
3208   if(opcode[i]==0x3a) // SWC2
3209   if(!(i_regs->waswritten&(1<<rs1[i]))&&!(new_dynarec_hacks&NDHACK_NO_SMC_CHECK)) {
3210 #if defined(HOST_IMM8)
3211     int ir=get_reg(i_regs->regmap,INVCP);
3212     assert(ir>=0);
3213     emit_cmpmem_indexedsr12_reg(ir,ar,1);
3214 #else
3215     emit_cmpmem_indexedsr12_imm(invalid_code,ar,1);
3216 #endif
3217     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3218     emit_callne(invalidate_addr_reg[ar]);
3219     #else
3220     void *jaddr3 = out;
3221     emit_jne(0);
3222     add_stub(INVCODE_STUB,jaddr3,out,reglist|(1<<HOST_CCREG),ar,0,0,0);
3223     #endif
3224   }
3225   if (opcode[i]==0x32) { // LWC2
3226     cop2_put_dreg(copr,tl,HOST_TEMPREG);
3227   }
3228 }
3229
3230 static void cop2_assemble(int i,struct regstat *i_regs)
3231 {
3232   u_int copr=(source[i]>>11)&0x1f;
3233   signed char temp=get_reg(i_regs->regmap,-1);
3234   if (opcode2[i]==0) { // MFC2
3235     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3236     if(tl>=0&&rt1[i]!=0)
3237       cop2_get_dreg(copr,tl,temp);
3238   }
3239   else if (opcode2[i]==4) { // MTC2
3240     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3241     cop2_put_dreg(copr,sl,temp);
3242   }
3243   else if (opcode2[i]==2) // CFC2
3244   {
3245     signed char tl=get_reg(i_regs->regmap,rt1[i]);
3246     if(tl>=0&&rt1[i]!=0)
3247       emit_readword(&reg_cop2c[copr],tl);
3248   }
3249   else if (opcode2[i]==6) // CTC2
3250   {
3251     signed char sl=get_reg(i_regs->regmap,rs1[i]);
3252     switch(copr) {
3253       case 4:
3254       case 12:
3255       case 20:
3256       case 26:
3257       case 27:
3258       case 29:
3259       case 30:
3260         emit_signextend16(sl,temp);
3261         break;
3262       case 31:
3263         //value = value & 0x7ffff000;
3264         //if (value & 0x7f87e000) value |= 0x80000000;
3265         emit_shrimm(sl,12,temp);
3266         emit_shlimm(temp,12,temp);
3267         emit_testimm(temp,0x7f000000);
3268         emit_testeqimm(temp,0x00870000);
3269         emit_testeqimm(temp,0x0000e000);
3270         emit_orrne_imm(temp,0x80000000,temp);
3271         break;
3272       default:
3273         temp=sl;
3274         break;
3275     }
3276     emit_writeword(temp,&reg_cop2c[copr]);
3277     assert(sl>=0);
3278   }
3279 }
3280
3281 #ifndef multdiv_assemble
3282 void multdiv_assemble(int i,struct regstat *i_regs)
3283 {
3284   printf("Need multdiv_assemble for this architecture.\n");
3285   exit(1);
3286 }
3287 #endif
3288
3289 void mov_assemble(int i,struct regstat *i_regs)
3290 {
3291   //if(opcode2[i]==0x10||opcode2[i]==0x12) { // MFHI/MFLO
3292   //if(opcode2[i]==0x11||opcode2[i]==0x13) { // MTHI/MTLO
3293   if(rt1[i]) {
3294     signed char sh,sl,th,tl;
3295     th=get_reg(i_regs->regmap,rt1[i]|64);
3296     tl=get_reg(i_regs->regmap,rt1[i]);
3297     //assert(tl>=0);
3298     if(tl>=0) {
3299       sh=get_reg(i_regs->regmap,rs1[i]|64);
3300       sl=get_reg(i_regs->regmap,rs1[i]);
3301       if(sl>=0) emit_mov(sl,tl);
3302       else emit_loadreg(rs1[i],tl);
3303       if(th>=0) {
3304         if(sh>=0) emit_mov(sh,th);
3305         else emit_loadreg(rs1[i]|64,th);
3306       }
3307     }
3308   }
3309 }
3310
3311 void syscall_assemble(int i,struct regstat *i_regs)
3312 {
3313   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3314   assert(ccreg==HOST_CCREG);
3315   assert(!is_delayslot);
3316   (void)ccreg;
3317   emit_movimm(start+i*4,EAX); // Get PC
3318   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3319   emit_jmp(jump_syscall_hle); // XXX
3320 }
3321
3322 void hlecall_assemble(int i,struct regstat *i_regs)
3323 {
3324   extern void psxNULL();
3325   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3326   assert(ccreg==HOST_CCREG);
3327   assert(!is_delayslot);
3328   (void)ccreg;
3329   emit_movimm(start+i*4+4,0); // Get PC
3330   uint32_t hleCode = source[i] & 0x03ffffff;
3331   if (hleCode >= ARRAY_SIZE(psxHLEt))
3332     emit_movimm((uintptr_t)psxNULL,1);
3333   else
3334     emit_movimm((uintptr_t)psxHLEt[hleCode],1);
3335   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG); // XXX
3336   emit_jmp(jump_hlecall);
3337 }
3338
3339 void intcall_assemble(int i,struct regstat *i_regs)
3340 {
3341   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3342   assert(ccreg==HOST_CCREG);
3343   assert(!is_delayslot);
3344   (void)ccreg;
3345   emit_movimm(start+i*4,0); // Get PC
3346   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]),HOST_CCREG);
3347   emit_jmp(jump_intcall);
3348 }
3349
3350 static void speculate_mov(int rs,int rt)
3351 {
3352   if(rt!=0) {
3353     smrv_strong_next|=1<<rt;
3354     smrv[rt]=smrv[rs];
3355   }
3356 }
3357
3358 static void speculate_mov_weak(int rs,int rt)
3359 {
3360   if(rt!=0) {
3361     smrv_weak_next|=1<<rt;
3362     smrv[rt]=smrv[rs];
3363   }
3364 }
3365
3366 static void speculate_register_values(int i)
3367 {
3368   if(i==0) {
3369     memcpy(smrv,psxRegs.GPR.r,sizeof(smrv));
3370     // gp,sp are likely to stay the same throughout the block
3371     smrv_strong_next=(1<<28)|(1<<29)|(1<<30);
3372     smrv_weak_next=~smrv_strong_next;
3373     //printf(" llr %08x\n", smrv[4]);
3374   }
3375   smrv_strong=smrv_strong_next;
3376   smrv_weak=smrv_weak_next;
3377   switch(itype[i]) {
3378     case ALU:
3379       if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3380       else if((smrv_strong>>rs2[i])&1) speculate_mov(rs2[i],rt1[i]);
3381       else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3382       else if((smrv_weak>>rs2[i])&1) speculate_mov_weak(rs2[i],rt1[i]);
3383       else {
3384         smrv_strong_next&=~(1<<rt1[i]);
3385         smrv_weak_next&=~(1<<rt1[i]);
3386       }
3387       break;
3388     case SHIFTIMM:
3389       smrv_strong_next&=~(1<<rt1[i]);
3390       smrv_weak_next&=~(1<<rt1[i]);
3391       // fallthrough
3392     case IMM16:
3393       if(rt1[i]&&is_const(&regs[i],rt1[i])) {
3394         int value,hr=get_reg(regs[i].regmap,rt1[i]);
3395         if(hr>=0) {
3396           if(get_final_value(hr,i,&value))
3397                smrv[rt1[i]]=value;
3398           else smrv[rt1[i]]=constmap[i][hr];
3399           smrv_strong_next|=1<<rt1[i];
3400         }
3401       }
3402       else {
3403         if     ((smrv_strong>>rs1[i])&1) speculate_mov(rs1[i],rt1[i]);
3404         else if((smrv_weak>>rs1[i])&1) speculate_mov_weak(rs1[i],rt1[i]);
3405       }
3406       break;
3407     case LOAD:
3408       if(start<0x2000&&(rt1[i]==26||(smrv[rt1[i]]>>24)==0xa0)) {
3409         // special case for BIOS
3410         smrv[rt1[i]]=0xa0000000;
3411         smrv_strong_next|=1<<rt1[i];
3412         break;
3413       }
3414       // fallthrough
3415     case SHIFT:
3416     case LOADLR:
3417     case MOV:
3418       smrv_strong_next&=~(1<<rt1[i]);
3419       smrv_weak_next&=~(1<<rt1[i]);
3420       break;
3421     case COP0:
3422     case COP2:
3423       if(opcode2[i]==0||opcode2[i]==2) { // MFC/CFC
3424         smrv_strong_next&=~(1<<rt1[i]);
3425         smrv_weak_next&=~(1<<rt1[i]);
3426       }
3427       break;
3428     case C2LS:
3429       if (opcode[i]==0x32) { // LWC2
3430         smrv_strong_next&=~(1<<rt1[i]);
3431         smrv_weak_next&=~(1<<rt1[i]);
3432       }
3433       break;
3434   }
3435 #if 0
3436   int r=4;
3437   printf("x %08x %08x %d %d c %08x %08x\n",smrv[r],start+i*4,
3438     ((smrv_strong>>r)&1),(smrv_weak>>r)&1,regs[i].isconst,regs[i].wasconst);
3439 #endif
3440 }
3441
3442 void ds_assemble(int i,struct regstat *i_regs)
3443 {
3444   speculate_register_values(i);
3445   is_delayslot=1;
3446   switch(itype[i]) {
3447     case ALU:
3448       alu_assemble(i,i_regs);break;
3449     case IMM16:
3450       imm16_assemble(i,i_regs);break;
3451     case SHIFT:
3452       shift_assemble(i,i_regs);break;
3453     case SHIFTIMM:
3454       shiftimm_assemble(i,i_regs);break;
3455     case LOAD:
3456       load_assemble(i,i_regs);break;
3457     case LOADLR:
3458       loadlr_assemble(i,i_regs);break;
3459     case STORE:
3460       store_assemble(i,i_regs);break;
3461     case STORELR:
3462       storelr_assemble(i,i_regs);break;
3463     case COP0:
3464       cop0_assemble(i,i_regs);break;
3465     case COP1:
3466       cop1_assemble(i,i_regs);break;
3467     case C1LS:
3468       c1ls_assemble(i,i_regs);break;
3469     case COP2:
3470       cop2_assemble(i,i_regs);break;
3471     case C2LS:
3472       c2ls_assemble(i,i_regs);break;
3473     case C2OP:
3474       c2op_assemble(i,i_regs);break;
3475     case MULTDIV:
3476       multdiv_assemble(i,i_regs);break;
3477     case MOV:
3478       mov_assemble(i,i_regs);break;
3479     case SYSCALL:
3480     case HLECALL:
3481     case INTCALL:
3482     case SPAN:
3483     case UJUMP:
3484     case RJUMP:
3485     case CJUMP:
3486     case SJUMP:
3487       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
3488   }
3489   is_delayslot=0;
3490 }
3491
3492 // Is the branch target a valid internal jump?
3493 static int internal_branch(int addr)
3494 {
3495   if(addr&1) return 0; // Indirect (register) jump
3496   if(addr>=start && addr<start+slen*4-4)
3497   {
3498     return 1;
3499   }
3500   return 0;
3501 }
3502
3503 static void wb_invalidate(signed char pre[],signed char entry[],uint64_t dirty,uint64_t u)
3504 {
3505   int hr;
3506   for(hr=0;hr<HOST_REGS;hr++) {
3507     if(hr!=EXCLUDE_REG) {
3508       if(pre[hr]!=entry[hr]) {
3509         if(pre[hr]>=0) {
3510           if((dirty>>hr)&1) {
3511             if(get_reg(entry,pre[hr])<0) {
3512               assert(pre[hr]<64);
3513               if(!((u>>pre[hr])&1))
3514                 emit_storereg(pre[hr],hr);
3515             }
3516           }
3517         }
3518       }
3519     }
3520   }
3521   // Move from one register to another (no writeback)
3522   for(hr=0;hr<HOST_REGS;hr++) {
3523     if(hr!=EXCLUDE_REG) {
3524       if(pre[hr]!=entry[hr]) {
3525         if(pre[hr]>=0&&(pre[hr]&63)<TEMPREG) {
3526           int nr;
3527           if((nr=get_reg(entry,pre[hr]))>=0) {
3528             emit_mov(hr,nr);
3529           }
3530         }
3531       }
3532     }
3533   }
3534 }
3535
3536 // Load the specified registers
3537 // This only loads the registers given as arguments because
3538 // we don't want to load things that will be overwritten
3539 static void load_regs(signed char entry[],signed char regmap[],int rs1,int rs2)
3540 {
3541   int hr;
3542   // Load 32-bit regs
3543   for(hr=0;hr<HOST_REGS;hr++) {
3544     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3545       if(entry[hr]!=regmap[hr]) {
3546         if(regmap[hr]==rs1||regmap[hr]==rs2)
3547         {
3548           if(regmap[hr]==0) {
3549             emit_zeroreg(hr);
3550           }
3551           else
3552           {
3553             emit_loadreg(regmap[hr],hr);
3554           }
3555         }
3556       }
3557     }
3558   }
3559 }
3560
3561 // Load registers prior to the start of a loop
3562 // so that they are not loaded within the loop
3563 static void loop_preload(signed char pre[],signed char entry[])
3564 {
3565   int hr;
3566   for(hr=0;hr<HOST_REGS;hr++) {
3567     if(hr!=EXCLUDE_REG) {
3568       if(pre[hr]!=entry[hr]) {
3569         if(entry[hr]>=0) {
3570           if(get_reg(pre,entry[hr])<0) {
3571             assem_debug("loop preload:\n");
3572             //printf("loop preload: %d\n",hr);
3573             if(entry[hr]==0) {
3574               emit_zeroreg(hr);
3575             }
3576             else if(entry[hr]<TEMPREG)
3577             {
3578               emit_loadreg(entry[hr],hr);
3579             }
3580             else if(entry[hr]-64<TEMPREG)
3581             {
3582               emit_loadreg(entry[hr],hr);
3583             }
3584           }
3585         }
3586       }
3587     }
3588   }
3589 }
3590
3591 // Generate address for load/store instruction
3592 // goes to AGEN for writes, FTEMP for LOADLR and cop1/2 loads
3593 void address_generation(int i,struct regstat *i_regs,signed char entry[])
3594 {
3595   if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS||itype[i]==C2LS) {
3596     int ra=-1;
3597     int agr=AGEN1+(i&1);
3598     if(itype[i]==LOAD) {
3599       ra=get_reg(i_regs->regmap,rt1[i]);
3600       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3601       assert(ra>=0);
3602     }
3603     if(itype[i]==LOADLR) {
3604       ra=get_reg(i_regs->regmap,FTEMP);
3605     }
3606     if(itype[i]==STORE||itype[i]==STORELR) {
3607       ra=get_reg(i_regs->regmap,agr);
3608       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3609     }
3610     if(itype[i]==C1LS||itype[i]==C2LS) {
3611       if ((opcode[i]&0x3b)==0x31||(opcode[i]&0x3b)==0x32) // LWC1/LDC1/LWC2/LDC2
3612         ra=get_reg(i_regs->regmap,FTEMP);
3613       else { // SWC1/SDC1/SWC2/SDC2
3614         ra=get_reg(i_regs->regmap,agr);
3615         if(ra<0) ra=get_reg(i_regs->regmap,-1);
3616       }
3617     }
3618     int rs=get_reg(i_regs->regmap,rs1[i]);
3619     if(ra>=0) {
3620       int offset=imm[i];
3621       int c=(i_regs->wasconst>>rs)&1;
3622       if(rs1[i]==0) {
3623         // Using r0 as a base address
3624         if(!entry||entry[ra]!=agr) {
3625           if (opcode[i]==0x22||opcode[i]==0x26) {
3626             emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3627           }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3628             emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3629           }else{
3630             emit_movimm(offset,ra);
3631           }
3632         } // else did it in the previous cycle
3633       }
3634       else if(rs<0) {
3635         if(!entry||entry[ra]!=rs1[i])
3636           emit_loadreg(rs1[i],ra);
3637         //if(!entry||entry[ra]!=rs1[i])
3638         //  printf("poor load scheduling!\n");
3639       }
3640       else if(c) {
3641         if(rs1[i]!=rt1[i]||itype[i]!=LOAD) {
3642           if(!entry||entry[ra]!=agr) {
3643             if (opcode[i]==0x22||opcode[i]==0x26) {
3644               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3645             }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3646               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3647             }else{
3648               emit_movimm(constmap[i][rs]+offset,ra);
3649               regs[i].loadedconst|=1<<ra;
3650             }
3651           } // else did it in the previous cycle
3652         } // else load_consts already did it
3653       }
3654       if(offset&&!c&&rs1[i]) {
3655         if(rs>=0) {
3656           emit_addimm(rs,offset,ra);
3657         }else{
3658           emit_addimm(ra,offset,ra);
3659         }
3660       }
3661     }
3662   }
3663   // Preload constants for next instruction
3664   if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS||itype[i+1]==C2LS) {
3665     int agr,ra;
3666     // Actual address
3667     agr=AGEN1+((i+1)&1);
3668     ra=get_reg(i_regs->regmap,agr);
3669     if(ra>=0) {
3670       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
3671       int offset=imm[i+1];
3672       int c=(regs[i+1].wasconst>>rs)&1;
3673       if(c&&(rs1[i+1]!=rt1[i+1]||itype[i+1]!=LOAD)) {
3674         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3675           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFFC,ra); // LWL/LWR
3676         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3677           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFF8,ra); // LDL/LDR
3678         }else{
3679           emit_movimm(constmap[i+1][rs]+offset,ra);
3680           regs[i+1].loadedconst|=1<<ra;
3681         }
3682       }
3683       else if(rs1[i+1]==0) {
3684         // Using r0 as a base address
3685         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
3686           emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3687         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
3688           emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3689         }else{
3690           emit_movimm(offset,ra);
3691         }
3692       }
3693     }
3694   }
3695 }
3696
3697 static int get_final_value(int hr, int i, int *value)
3698 {
3699   int reg=regs[i].regmap[hr];
3700   while(i<slen-1) {
3701     if(regs[i+1].regmap[hr]!=reg) break;
3702     if(!((regs[i+1].isconst>>hr)&1)) break;
3703     if(bt[i+1]) break;
3704     i++;
3705   }
3706   if(i<slen-1) {
3707     if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
3708       *value=constmap[i][hr];
3709       return 1;
3710     }
3711     if(!bt[i+1]) {
3712       if(itype[i+1]==UJUMP||itype[i+1]==RJUMP||itype[i+1]==CJUMP||itype[i+1]==SJUMP) {
3713         // Load in delay slot, out-of-order execution
3714         if(itype[i+2]==LOAD&&rs1[i+2]==reg&&rt1[i+2]==reg&&((regs[i+1].wasconst>>hr)&1))
3715         {
3716           // Precompute load address
3717           *value=constmap[i][hr]+imm[i+2];
3718           return 1;
3719         }
3720       }
3721       if(itype[i+1]==LOAD&&rs1[i+1]==reg&&rt1[i+1]==reg)
3722       {
3723         // Precompute load address
3724         *value=constmap[i][hr]+imm[i+1];
3725         //printf("c=%x imm=%lx\n",(long)constmap[i][hr],imm[i+1]);
3726         return 1;
3727       }
3728     }
3729   }
3730   *value=constmap[i][hr];
3731   //printf("c=%lx\n",(long)constmap[i][hr]);
3732   if(i==slen-1) return 1;
3733   assert(reg < 64);
3734   return !((unneeded_reg[i+1]>>reg)&1);
3735 }
3736
3737 // Load registers with known constants
3738 static void load_consts(signed char pre[],signed char regmap[],int i)
3739 {
3740   int hr,hr2;
3741   // propagate loaded constant flags
3742   if(i==0||bt[i])
3743     regs[i].loadedconst=0;
3744   else {
3745     for(hr=0;hr<HOST_REGS;hr++) {
3746       if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((regs[i-1].isconst>>hr)&1)&&pre[hr]==regmap[hr]
3747          &&regmap[hr]==regs[i-1].regmap[hr]&&((regs[i-1].loadedconst>>hr)&1))
3748       {
3749         regs[i].loadedconst|=1<<hr;
3750       }
3751     }
3752   }
3753   // Load 32-bit regs
3754   for(hr=0;hr<HOST_REGS;hr++) {
3755     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3756       //if(entry[hr]!=regmap[hr]) {
3757       if(!((regs[i].loadedconst>>hr)&1)) {
3758         assert(regmap[hr]<64);
3759         if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
3760           int value,similar=0;
3761           if(get_final_value(hr,i,&value)) {
3762             // see if some other register has similar value
3763             for(hr2=0;hr2<HOST_REGS;hr2++) {
3764               if(hr2!=EXCLUDE_REG&&((regs[i].loadedconst>>hr2)&1)) {
3765                 if(is_similar_value(value,constmap[i][hr2])) {
3766                   similar=1;
3767                   break;
3768                 }
3769               }
3770             }
3771             if(similar) {
3772               int value2;
3773               if(get_final_value(hr2,i,&value2)) // is this needed?
3774                 emit_movimm_from(value2,hr2,value,hr);
3775               else
3776                 emit_movimm(value,hr);
3777             }
3778             else if(value==0) {
3779               emit_zeroreg(hr);
3780             }
3781             else {
3782               emit_movimm(value,hr);
3783             }
3784           }
3785           regs[i].loadedconst|=1<<hr;
3786         }
3787       }
3788     }
3789   }
3790 }
3791
3792 void load_all_consts(signed char regmap[], u_int dirty, int i)
3793 {
3794   int hr;
3795   // Load 32-bit regs
3796   for(hr=0;hr<HOST_REGS;hr++) {
3797     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
3798       assert(regmap[hr] < 64);
3799       if(((regs[i].isconst>>hr)&1)&&regmap[hr]>0) {
3800         int value=constmap[i][hr];
3801         if(value==0) {
3802           emit_zeroreg(hr);
3803         }
3804         else {
3805           emit_movimm(value,hr);
3806         }
3807       }
3808     }
3809   }
3810 }
3811
3812 // Write out all dirty registers (except cycle count)
3813 static void wb_dirtys(signed char i_regmap[],uint64_t i_dirty)
3814 {
3815   int hr;
3816   for(hr=0;hr<HOST_REGS;hr++) {
3817     if(hr!=EXCLUDE_REG) {
3818       if(i_regmap[hr]>0) {
3819         if(i_regmap[hr]!=CCREG) {
3820           if((i_dirty>>hr)&1) {
3821             assert(i_regmap[hr]<64);
3822             emit_storereg(i_regmap[hr],hr);
3823           }
3824         }
3825       }
3826     }
3827   }
3828 }
3829
3830 // Write out dirty registers that we need to reload (pair with load_needed_regs)
3831 // This writes the registers not written by store_regs_bt
3832 void wb_needed_dirtys(signed char i_regmap[],uint64_t i_dirty,int addr)
3833 {
3834   int hr;
3835   int t=(addr-start)>>2;
3836   for(hr=0;hr<HOST_REGS;hr++) {
3837     if(hr!=EXCLUDE_REG) {
3838       if(i_regmap[hr]>0) {
3839         if(i_regmap[hr]!=CCREG) {
3840           if(i_regmap[hr]==regs[t].regmap_entry[hr] && ((regs[t].dirty>>hr)&1)) {
3841             if((i_dirty>>hr)&1) {
3842               assert(i_regmap[hr]<64);
3843               emit_storereg(i_regmap[hr],hr);
3844             }
3845           }
3846         }
3847       }
3848     }
3849   }
3850 }
3851
3852 // Load all registers (except cycle count)
3853 void load_all_regs(signed char i_regmap[])
3854 {
3855   int hr;
3856   for(hr=0;hr<HOST_REGS;hr++) {
3857     if(hr!=EXCLUDE_REG) {
3858       if(i_regmap[hr]==0) {
3859         emit_zeroreg(hr);
3860       }
3861       else
3862       if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
3863       {
3864         emit_loadreg(i_regmap[hr],hr);
3865       }
3866     }
3867   }
3868 }
3869
3870 // Load all current registers also needed by next instruction
3871 void load_needed_regs(signed char i_regmap[],signed char next_regmap[])
3872 {
3873   int hr;
3874   for(hr=0;hr<HOST_REGS;hr++) {
3875     if(hr!=EXCLUDE_REG) {
3876       if(get_reg(next_regmap,i_regmap[hr])>=0) {
3877         if(i_regmap[hr]==0) {
3878           emit_zeroreg(hr);
3879         }
3880         else
3881         if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
3882         {
3883           emit_loadreg(i_regmap[hr],hr);
3884         }
3885       }
3886     }
3887   }
3888 }
3889
3890 // Load all regs, storing cycle count if necessary
3891 void load_regs_entry(int t)
3892 {
3893   int hr;
3894   if(is_ds[t]) emit_addimm(HOST_CCREG,CLOCK_ADJUST(1),HOST_CCREG);
3895   else if(ccadj[t]) emit_addimm(HOST_CCREG,-CLOCK_ADJUST(ccadj[t]),HOST_CCREG);
3896   if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
3897     emit_storereg(CCREG,HOST_CCREG);
3898   }
3899   // Load 32-bit regs
3900   for(hr=0;hr<HOST_REGS;hr++) {
3901     if(regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
3902       if(regs[t].regmap_entry[hr]==0) {
3903         emit_zeroreg(hr);
3904       }
3905       else if(regs[t].regmap_entry[hr]!=CCREG)
3906       {
3907         emit_loadreg(regs[t].regmap_entry[hr],hr);
3908       }
3909     }
3910   }
3911 }
3912
3913 // Store dirty registers prior to branch
3914 void store_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3915 {
3916   if(internal_branch(addr))
3917   {
3918     int t=(addr-start)>>2;
3919     int hr;
3920     for(hr=0;hr<HOST_REGS;hr++) {
3921       if(hr!=EXCLUDE_REG) {
3922         if(i_regmap[hr]>0 && i_regmap[hr]!=CCREG) {
3923           if(i_regmap[hr]!=regs[t].regmap_entry[hr] || !((regs[t].dirty>>hr)&1)) {
3924             if((i_dirty>>hr)&1) {
3925               assert(i_regmap[hr]<64);
3926               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
3927                 emit_storereg(i_regmap[hr],hr);
3928             }
3929           }
3930         }
3931       }
3932     }
3933   }
3934   else
3935   {
3936     // Branch out of this block, write out all dirty regs
3937     wb_dirtys(i_regmap,i_dirty);
3938   }
3939 }
3940
3941 // Load all needed registers for branch target
3942 static void load_regs_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3943 {
3944   //if(addr>=start && addr<(start+slen*4))
3945   if(internal_branch(addr))
3946   {
3947     int t=(addr-start)>>2;
3948     int hr;
3949     // Store the cycle count before loading something else
3950     if(i_regmap[HOST_CCREG]!=CCREG) {
3951       assert(i_regmap[HOST_CCREG]==-1);
3952     }
3953     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
3954       emit_storereg(CCREG,HOST_CCREG);
3955     }
3956     // Load 32-bit regs
3957     for(hr=0;hr<HOST_REGS;hr++) {
3958       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
3959         if(i_regmap[hr]!=regs[t].regmap_entry[hr]) {
3960           if(regs[t].regmap_entry[hr]==0) {
3961             emit_zeroreg(hr);
3962           }
3963           else if(regs[t].regmap_entry[hr]!=CCREG)
3964           {
3965             emit_loadreg(regs[t].regmap_entry[hr],hr);
3966           }
3967         }
3968       }
3969     }
3970   }
3971 }
3972
3973 static int match_bt(signed char i_regmap[],uint64_t i_dirty,int addr)
3974 {
3975   if(addr>=start && addr<start+slen*4-4)
3976   {
3977     int t=(addr-start)>>2;
3978     int hr;
3979     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) return 0;
3980     for(hr=0;hr<HOST_REGS;hr++)
3981     {
3982       if(hr!=EXCLUDE_REG)
3983       {
3984         if(i_regmap[hr]!=regs[t].regmap_entry[hr])
3985         {
3986           if(regs[t].regmap_entry[hr]>=0&&(regs[t].regmap_entry[hr]|64)<TEMPREG+64)
3987           {
3988             return 0;
3989           }
3990           else
3991           if((i_dirty>>hr)&1)
3992           {
3993             if(i_regmap[hr]<TEMPREG)
3994             {
3995               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
3996                 return 0;
3997             }
3998             else if(i_regmap[hr]>=64&&i_regmap[hr]<TEMPREG+64)
3999             {
4000               assert(0);
4001             }
4002           }
4003         }
4004         else // Same register but is it 32-bit or dirty?
4005         if(i_regmap[hr]>=0)
4006         {
4007           if(!((regs[t].dirty>>hr)&1))
4008           {
4009             if((i_dirty>>hr)&1)
4010             {
4011               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4012               {
4013                 //printf("%x: dirty no match\n",addr);
4014                 return 0;
4015               }
4016             }
4017           }
4018         }
4019       }
4020     }
4021     // Delay slots are not valid branch targets
4022     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP)) return 0;
4023     // Delay slots require additional processing, so do not match
4024     if(is_ds[t]) return 0;
4025   }
4026   else
4027   {
4028     int hr;
4029     for(hr=0;hr<HOST_REGS;hr++)
4030     {
4031       if(hr!=EXCLUDE_REG)
4032       {
4033         if(i_regmap[hr]>=0)
4034         {
4035           if(hr!=HOST_CCREG||i_regmap[hr]!=CCREG)
4036           {
4037             if((i_dirty>>hr)&1)
4038             {
4039               return 0;
4040             }
4041           }
4042         }
4043       }
4044     }
4045   }
4046   return 1;
4047 }
4048
4049 #ifdef DRC_DBG
4050 static void drc_dbg_emit_do_cmp(int i)
4051 {
4052   extern void do_insn_cmp();
4053   extern int cycle;
4054   u_int hr,reglist=0;
4055
4056   for(hr=0;hr<HOST_REGS;hr++)
4057     if(regs[i].regmap[hr]>=0) reglist|=1<<hr;
4058   save_regs(reglist);
4059   emit_movimm(start+i*4,0);
4060   emit_writeword(0,&pcaddr);
4061   emit_call(do_insn_cmp);
4062   //emit_readword(&cycle,0);
4063   //emit_addimm(0,2,0);
4064   //emit_writeword(0,&cycle);
4065   restore_regs(reglist);
4066 }
4067 #else
4068 #define drc_dbg_emit_do_cmp(x)
4069 #endif
4070
4071 // Used when a branch jumps into the delay slot of another branch
4072 void ds_assemble_entry(int i)
4073 {
4074   int t=(ba[i]-start)>>2;
4075   if (!instr_addr[t])
4076     instr_addr[t] = out;
4077   assem_debug("Assemble delay slot at %x\n",ba[i]);
4078   assem_debug("<->\n");
4079   drc_dbg_emit_do_cmp(t);
4080   if(regs[t].regmap_entry[HOST_CCREG]==CCREG&&regs[t].regmap[HOST_CCREG]!=CCREG)
4081     wb_register(CCREG,regs[t].regmap_entry,regs[t].wasdirty);
4082   load_regs(regs[t].regmap_entry,regs[t].regmap,rs1[t],rs2[t]);
4083   address_generation(t,&regs[t],regs[t].regmap_entry);
4084   if(itype[t]==STORE||itype[t]==STORELR||(opcode[t]&0x3b)==0x39||(opcode[t]&0x3b)==0x3a)
4085     load_regs(regs[t].regmap_entry,regs[t].regmap,INVCP,INVCP);
4086   is_delayslot=0;
4087   switch(itype[t]) {
4088     case ALU:
4089       alu_assemble(t,&regs[t]);break;
4090     case IMM16:
4091       imm16_assemble(t,&regs[t]);break;
4092     case SHIFT:
4093       shift_assemble(t,&regs[t]);break;
4094     case SHIFTIMM:
4095       shiftimm_assemble(t,&regs[t]);break;
4096     case LOAD:
4097       load_assemble(t,&regs[t]);break;
4098     case LOADLR:
4099       loadlr_assemble(t,&regs[t]);break;
4100     case STORE:
4101       store_assemble(t,&regs[t]);break;
4102     case STORELR:
4103       storelr_assemble(t,&regs[t]);break;
4104     case COP0:
4105       cop0_assemble(t,&regs[t]);break;
4106     case COP1:
4107       cop1_assemble(t,&regs[t]);break;
4108     case C1LS:
4109       c1ls_assemble(t,&regs[t]);break;
4110     case COP2:
4111       cop2_assemble(t,&regs[t]);break;
4112     case C2LS:
4113       c2ls_assemble(t,&regs[t]);break;
4114     case C2OP:
4115       c2op_assemble(t,&regs[t]);break;
4116     case MULTDIV:
4117       multdiv_assemble(t,&regs[t]);break;
4118     case MOV:
4119       mov_assemble(t,&regs[t]);break;
4120     case SYSCALL:
4121     case HLECALL:
4122     case INTCALL:
4123     case SPAN:
4124     case UJUMP:
4125     case RJUMP:
4126     case CJUMP:
4127     case SJUMP:
4128       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
4129   }
4130   store_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4131   load_regs_bt(regs[t].regmap,regs[t].dirty,ba[i]+4);
4132   if(internal_branch(ba[i]+4))
4133     assem_debug("branch: internal\n");
4134   else
4135     assem_debug("branch: external\n");
4136   assert(internal_branch(ba[i]+4));
4137   add_to_linker(out,ba[i]+4,internal_branch(ba[i]+4));
4138   emit_jmp(0);
4139 }
4140
4141 void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert)
4142 {
4143   int count;
4144   void *jaddr;
4145   void *idle=NULL;
4146   int t=0;
4147   if(itype[i]==RJUMP)
4148   {
4149     *adj=0;
4150   }
4151   //if(ba[i]>=start && ba[i]<(start+slen*4))
4152   if(internal_branch(ba[i]))
4153   {
4154     t=(ba[i]-start)>>2;
4155     if(is_ds[t]) *adj=-1; // Branch into delay slot adds an extra cycle
4156     else *adj=ccadj[t];
4157   }
4158   else
4159   {
4160     *adj=0;
4161   }
4162   count=ccadj[i];
4163   if(taken==TAKEN && i==(ba[i]-start)>>2 && source[i+1]==0) {
4164     // Idle loop
4165     if(count&1) emit_addimm_and_set_flags(2*(count+2),HOST_CCREG);
4166     idle=out;
4167     //emit_subfrommem(&idlecount,HOST_CCREG); // Count idle cycles
4168     emit_andimm(HOST_CCREG,3,HOST_CCREG);
4169     jaddr=out;
4170     emit_jmp(0);
4171   }
4172   else if(*adj==0||invert) {
4173     int cycles=CLOCK_ADJUST(count+2);
4174     // faster loop HACK
4175     if (t&&*adj) {
4176       int rel=t-i;
4177       if(-NO_CYCLE_PENALTY_THR<rel&&rel<0)
4178         cycles=CLOCK_ADJUST(*adj)+count+2-*adj;
4179     }
4180     emit_addimm_and_set_flags(cycles,HOST_CCREG);
4181     jaddr=out;
4182     emit_jns(0);
4183   }
4184   else
4185   {
4186     emit_cmpimm(HOST_CCREG,-CLOCK_ADJUST(count+2));
4187     jaddr=out;
4188     emit_jns(0);
4189   }
4190   add_stub(CC_STUB,jaddr,idle?idle:out,(*adj==0||invert||idle)?0:(count+2),i,addr,taken,0);
4191 }
4192
4193 static void do_ccstub(int n)
4194 {
4195   literal_pool(256);
4196   assem_debug("do_ccstub %x\n",start+stubs[n].b*4);
4197   set_jump_target(stubs[n].addr, out);
4198   int i=stubs[n].b;
4199   if(stubs[n].d==NULLDS) {
4200     // Delay slot instruction is nullified ("likely" branch)
4201     wb_dirtys(regs[i].regmap,regs[i].dirty);
4202   }
4203   else if(stubs[n].d!=TAKEN) {
4204     wb_dirtys(branch_regs[i].regmap,branch_regs[i].dirty);
4205   }
4206   else {
4207     if(internal_branch(ba[i]))
4208       wb_needed_dirtys(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4209   }
4210   if(stubs[n].c!=-1)
4211   {
4212     // Save PC as return address
4213     emit_movimm(stubs[n].c,EAX);
4214     emit_writeword(EAX,&pcaddr);
4215   }
4216   else
4217   {
4218     // Return address depends on which way the branch goes
4219     if(itype[i]==CJUMP||itype[i]==SJUMP)
4220     {
4221       int s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4222       int s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4223       if(rs1[i]==0)
4224       {
4225         s1l=s2l;
4226         s2l=-1;
4227       }
4228       else if(rs2[i]==0)
4229       {
4230         s2l=-1;
4231       }
4232       assert(s1l>=0);
4233       #ifdef DESTRUCTIVE_WRITEBACK
4234       if(rs1[i]) {
4235         if((branch_regs[i].dirty>>s1l)&&1)
4236           emit_loadreg(rs1[i],s1l);
4237       }
4238       else {
4239         if((branch_regs[i].dirty>>s1l)&1)
4240           emit_loadreg(rs2[i],s1l);
4241       }
4242       if(s2l>=0)
4243         if((branch_regs[i].dirty>>s2l)&1)
4244           emit_loadreg(rs2[i],s2l);
4245       #endif
4246       int hr=0;
4247       int addr=-1,alt=-1,ntaddr=-1;
4248       while(hr<HOST_REGS)
4249       {
4250         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4251            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4252            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4253         {
4254           addr=hr++;break;
4255         }
4256         hr++;
4257       }
4258       while(hr<HOST_REGS)
4259       {
4260         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4261            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4262            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4263         {
4264           alt=hr++;break;
4265         }
4266         hr++;
4267       }
4268       if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
4269       {
4270         while(hr<HOST_REGS)
4271         {
4272           if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4273              (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4274              (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4275           {
4276             ntaddr=hr;break;
4277           }
4278           hr++;
4279         }
4280         assert(hr<HOST_REGS);
4281       }
4282       if((opcode[i]&0x2f)==4) // BEQ
4283       {
4284         #ifdef HAVE_CMOV_IMM
4285         if(s2l>=0) emit_cmp(s1l,s2l);
4286         else emit_test(s1l,s1l);
4287         emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
4288         #else
4289         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4290         if(s2l>=0) emit_cmp(s1l,s2l);
4291         else emit_test(s1l,s1l);
4292         emit_cmovne_reg(alt,addr);
4293         #endif
4294       }
4295       if((opcode[i]&0x2f)==5) // BNE
4296       {
4297         #ifdef HAVE_CMOV_IMM
4298         if(s2l>=0) emit_cmp(s1l,s2l);
4299         else emit_test(s1l,s1l);
4300         emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
4301         #else
4302         emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
4303         if(s2l>=0) emit_cmp(s1l,s2l);
4304         else emit_test(s1l,s1l);
4305         emit_cmovne_reg(alt,addr);
4306         #endif
4307       }
4308       if((opcode[i]&0x2f)==6) // BLEZ
4309       {
4310         //emit_movimm(ba[i],alt);
4311         //emit_movimm(start+i*4+8,addr);
4312         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4313         emit_cmpimm(s1l,1);
4314         emit_cmovl_reg(alt,addr);
4315       }
4316       if((opcode[i]&0x2f)==7) // BGTZ
4317       {
4318         //emit_movimm(ba[i],addr);
4319         //emit_movimm(start+i*4+8,ntaddr);
4320         emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
4321         emit_cmpimm(s1l,1);
4322         emit_cmovl_reg(ntaddr,addr);
4323       }
4324       if((opcode[i]==1)&&(opcode2[i]&0x2D)==0) // BLTZ
4325       {
4326         //emit_movimm(ba[i],alt);
4327         //emit_movimm(start+i*4+8,addr);
4328         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4329         emit_test(s1l,s1l);
4330         emit_cmovs_reg(alt,addr);
4331       }
4332       if((opcode[i]==1)&&(opcode2[i]&0x2D)==1) // BGEZ
4333       {
4334         //emit_movimm(ba[i],addr);
4335         //emit_movimm(start+i*4+8,alt);
4336         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4337         emit_test(s1l,s1l);
4338         emit_cmovs_reg(alt,addr);
4339       }
4340       if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
4341         if(source[i]&0x10000) // BC1T
4342         {
4343           //emit_movimm(ba[i],alt);
4344           //emit_movimm(start+i*4+8,addr);
4345           emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4346           emit_testimm(s1l,0x800000);
4347           emit_cmovne_reg(alt,addr);
4348         }
4349         else // BC1F
4350         {
4351           //emit_movimm(ba[i],addr);
4352           //emit_movimm(start+i*4+8,alt);
4353           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4354           emit_testimm(s1l,0x800000);
4355           emit_cmovne_reg(alt,addr);
4356         }
4357       }
4358       emit_writeword(addr,&pcaddr);
4359     }
4360     else
4361     if(itype[i]==RJUMP)
4362     {
4363       int r=get_reg(branch_regs[i].regmap,rs1[i]);
4364       if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4365         r=get_reg(branch_regs[i].regmap,RTEMP);
4366       }
4367       emit_writeword(r,&pcaddr);
4368     }
4369     else {SysPrintf("Unknown branch type in do_ccstub\n");exit(1);}
4370   }
4371   // Update cycle count
4372   assert(branch_regs[i].regmap[HOST_CCREG]==CCREG||branch_regs[i].regmap[HOST_CCREG]==-1);
4373   if(stubs[n].a) emit_addimm(HOST_CCREG,CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4374   emit_call(cc_interrupt);
4375   if(stubs[n].a) emit_addimm(HOST_CCREG,-CLOCK_ADJUST((signed int)stubs[n].a),HOST_CCREG);
4376   if(stubs[n].d==TAKEN) {
4377     if(internal_branch(ba[i]))
4378       load_needed_regs(branch_regs[i].regmap,regs[(ba[i]-start)>>2].regmap_entry);
4379     else if(itype[i]==RJUMP) {
4380       if(get_reg(branch_regs[i].regmap,RTEMP)>=0)
4381         emit_readword(&pcaddr,get_reg(branch_regs[i].regmap,RTEMP));
4382       else
4383         emit_loadreg(rs1[i],get_reg(branch_regs[i].regmap,rs1[i]));
4384     }
4385   }else if(stubs[n].d==NOTTAKEN) {
4386     if(i<slen-2) load_needed_regs(branch_regs[i].regmap,regmap_pre[i+2]);
4387     else load_all_regs(branch_regs[i].regmap);
4388   }else if(stubs[n].d==NULLDS) {
4389     // Delay slot instruction is nullified ("likely" branch)
4390     if(i<slen-2) load_needed_regs(regs[i].regmap,regmap_pre[i+2]);
4391     else load_all_regs(regs[i].regmap);
4392   }else{
4393     load_all_regs(branch_regs[i].regmap);
4394   }
4395   emit_jmp(stubs[n].retaddr);
4396 }
4397
4398 static void add_to_linker(void *addr, u_int target, int ext)
4399 {
4400   assert(linkcount < ARRAY_SIZE(link_addr));
4401   link_addr[linkcount].addr = addr;
4402   link_addr[linkcount].target = target;
4403   link_addr[linkcount].ext = ext;
4404   linkcount++;
4405 }
4406
4407 static void ujump_assemble_write_ra(int i)
4408 {
4409   int rt;
4410   unsigned int return_address;
4411   rt=get_reg(branch_regs[i].regmap,31);
4412   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4413   //assert(rt>=0);
4414   return_address=start+i*4+8;
4415   if(rt>=0) {
4416     #ifdef USE_MINI_HT
4417     if(internal_branch(return_address)&&rt1[i+1]!=31) {
4418       int temp=-1; // note: must be ds-safe
4419       #ifdef HOST_TEMPREG
4420       temp=HOST_TEMPREG;
4421       #endif
4422       if(temp>=0) do_miniht_insert(return_address,rt,temp);
4423       else emit_movimm(return_address,rt);
4424     }
4425     else
4426     #endif
4427     {
4428       #ifdef REG_PREFETCH
4429       if(temp>=0)
4430       {
4431         if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4432       }
4433       #endif
4434       emit_movimm(return_address,rt); // PC into link register
4435       #ifdef IMM_PREFETCH
4436       emit_prefetch(hash_table_get(return_address));
4437       #endif
4438     }
4439   }
4440 }
4441
4442 void ujump_assemble(int i,struct regstat *i_regs)
4443 {
4444   int ra_done=0;
4445   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4446   address_generation(i+1,i_regs,regs[i].regmap_entry);
4447   #ifdef REG_PREFETCH
4448   int temp=get_reg(branch_regs[i].regmap,PTEMP);
4449   if(rt1[i]==31&&temp>=0)
4450   {
4451     signed char *i_regmap=i_regs->regmap;
4452     int return_address=start+i*4+8;
4453     if(get_reg(branch_regs[i].regmap,31)>0)
4454     if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4455   }
4456   #endif
4457   if(rt1[i]==31&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4458     ujump_assemble_write_ra(i); // writeback ra for DS
4459     ra_done=1;
4460   }
4461   ds_assemble(i+1,i_regs);
4462   uint64_t bc_unneeded=branch_regs[i].u;
4463   bc_unneeded|=1|(1LL<<rt1[i]);
4464   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4465   load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4466   if(!ra_done&&rt1[i]==31)
4467     ujump_assemble_write_ra(i);
4468   int cc,adj;
4469   cc=get_reg(branch_regs[i].regmap,CCREG);
4470   assert(cc==HOST_CCREG);
4471   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4472   #ifdef REG_PREFETCH
4473   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4474   #endif
4475   do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4476   if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4477   load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4478   if(internal_branch(ba[i]))
4479     assem_debug("branch: internal\n");
4480   else
4481     assem_debug("branch: external\n");
4482   if(internal_branch(ba[i])&&is_ds[(ba[i]-start)>>2]) {
4483     ds_assemble_entry(i);
4484   }
4485   else {
4486     add_to_linker(out,ba[i],internal_branch(ba[i]));
4487     emit_jmp(0);
4488   }
4489 }
4490
4491 static void rjump_assemble_write_ra(int i)
4492 {
4493   int rt,return_address;
4494   assert(rt1[i+1]!=rt1[i]);
4495   assert(rt2[i+1]!=rt1[i]);
4496   rt=get_reg(branch_regs[i].regmap,rt1[i]);
4497   assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4498   assert(rt>=0);
4499   return_address=start+i*4+8;
4500   #ifdef REG_PREFETCH
4501   if(temp>=0)
4502   {
4503     if(i_regmap[temp]!=PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4504   }
4505   #endif
4506   emit_movimm(return_address,rt); // PC into link register
4507   #ifdef IMM_PREFETCH
4508   emit_prefetch(hash_table_get(return_address));
4509   #endif
4510 }
4511
4512 void rjump_assemble(int i,struct regstat *i_regs)
4513 {
4514   int temp;
4515   int rs,cc;
4516   int ra_done=0;
4517   rs=get_reg(branch_regs[i].regmap,rs1[i]);
4518   assert(rs>=0);
4519   if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4520     // Delay slot abuse, make a copy of the branch address register
4521     temp=get_reg(branch_regs[i].regmap,RTEMP);
4522     assert(temp>=0);
4523     assert(regs[i].regmap[temp]==RTEMP);
4524     emit_mov(rs,temp);
4525     rs=temp;
4526   }
4527   address_generation(i+1,i_regs,regs[i].regmap_entry);
4528   #ifdef REG_PREFETCH
4529   if(rt1[i]==31)
4530   {
4531     if((temp=get_reg(branch_regs[i].regmap,PTEMP))>=0) {
4532       signed char *i_regmap=i_regs->regmap;
4533       int return_address=start+i*4+8;
4534       if(i_regmap[temp]==PTEMP) emit_movimm((uintptr_t)hash_table_get(return_address),temp);
4535     }
4536   }
4537   #endif
4538   #ifdef USE_MINI_HT
4539   if(rs1[i]==31) {
4540     int rh=get_reg(regs[i].regmap,RHASH);
4541     if(rh>=0) do_preload_rhash(rh);
4542   }
4543   #endif
4544   if(rt1[i]!=0&&(rt1[i]==rs1[i+1]||rt1[i]==rs2[i+1])) {
4545     rjump_assemble_write_ra(i);
4546     ra_done=1;
4547   }
4548   ds_assemble(i+1,i_regs);
4549   uint64_t bc_unneeded=branch_regs[i].u;
4550   bc_unneeded|=1|(1LL<<rt1[i]);
4551   bc_unneeded&=~(1LL<<rs1[i]);
4552   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4553   load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],CCREG);
4554   if(!ra_done&&rt1[i]!=0)
4555     rjump_assemble_write_ra(i);
4556   cc=get_reg(branch_regs[i].regmap,CCREG);
4557   assert(cc==HOST_CCREG);
4558   (void)cc;
4559   #ifdef USE_MINI_HT
4560   int rh=get_reg(branch_regs[i].regmap,RHASH);
4561   int ht=get_reg(branch_regs[i].regmap,RHTBL);
4562   if(rs1[i]==31) {
4563     if(regs[i].regmap[rh]!=RHASH) do_preload_rhash(rh);
4564     do_preload_rhtbl(ht);
4565     do_rhash(rs,rh);
4566   }
4567   #endif
4568   store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4569   #ifdef DESTRUCTIVE_WRITEBACK
4570   if((branch_regs[i].dirty>>rs)&1) {
4571     if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
4572       emit_loadreg(rs1[i],rs);
4573     }
4574   }
4575   #endif
4576   #ifdef REG_PREFETCH
4577   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
4578   #endif
4579   #ifdef USE_MINI_HT
4580   if(rs1[i]==31) {
4581     do_miniht_load(ht,rh);
4582   }
4583   #endif
4584   //do_cc(i,branch_regs[i].regmap,&adj,-1,TAKEN);
4585   //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen
4586   //assert(adj==0);
4587   emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4588   add_stub(CC_STUB,out,jump_vaddr_reg[rs],0,i,-1,TAKEN,0);
4589   if(itype[i+1]==COP0&&(source[i+1]&0x3f)==0x10)
4590     // special case for RFE
4591     emit_jmp(0);
4592   else
4593     emit_jns(0);
4594   //load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,-1);
4595   #ifdef USE_MINI_HT
4596   if(rs1[i]==31) {
4597     do_miniht_jump(rs,rh,ht);
4598   }
4599   else
4600   #endif
4601   {
4602     emit_jmp(jump_vaddr_reg[rs]);
4603   }
4604   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4605   if(rt1[i]!=31&&i<slen-2&&(((u_int)out)&7)) emit_mov(13,13);
4606   #endif
4607 }
4608
4609 void cjump_assemble(int i,struct regstat *i_regs)
4610 {
4611   signed char *i_regmap=i_regs->regmap;
4612   int cc;
4613   int match;
4614   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4615   assem_debug("match=%d\n",match);
4616   int s1l,s2l;
4617   int unconditional=0,nop=0;
4618   int invert=0;
4619   int internal=internal_branch(ba[i]);
4620   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4621   if(!match) invert=1;
4622   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4623   if(i>(ba[i]-start)>>2) invert=1;
4624   #endif
4625
4626   if(ooo[i]) {
4627     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4628     s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4629   }
4630   else {
4631     s1l=get_reg(i_regmap,rs1[i]);
4632     s2l=get_reg(i_regmap,rs2[i]);
4633   }
4634   if(rs1[i]==0&&rs2[i]==0)
4635   {
4636     if(opcode[i]&1) nop=1;
4637     else unconditional=1;
4638     //assert(opcode[i]!=5);
4639     //assert(opcode[i]!=7);
4640     //assert(opcode[i]!=0x15);
4641     //assert(opcode[i]!=0x17);
4642   }
4643   else if(rs1[i]==0)
4644   {
4645     s1l=s2l;
4646     s2l=-1;
4647   }
4648   else if(rs2[i]==0)
4649   {
4650     s2l=-1;
4651   }
4652
4653   if(ooo[i]) {
4654     // Out of order execution (delay slot first)
4655     //printf("OOOE\n");
4656     address_generation(i+1,i_regs,regs[i].regmap_entry);
4657     ds_assemble(i+1,i_regs);
4658     int adj;
4659     uint64_t bc_unneeded=branch_regs[i].u;
4660     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
4661     bc_unneeded|=1;
4662     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4663     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs2[i]);
4664     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4665     cc=get_reg(branch_regs[i].regmap,CCREG);
4666     assert(cc==HOST_CCREG);
4667     if(unconditional)
4668       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4669     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
4670     //assem_debug("cycle count (adj)\n");
4671     if(unconditional) {
4672       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4673       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
4674         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4675         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4676         if(internal)
4677           assem_debug("branch: internal\n");
4678         else
4679           assem_debug("branch: external\n");
4680         if(internal&&is_ds[(ba[i]-start)>>2]) {
4681           ds_assemble_entry(i);
4682         }
4683         else {
4684           add_to_linker(out,ba[i],internal);
4685           emit_jmp(0);
4686         }
4687         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4688         if(((u_int)out)&7) emit_addnop(0);
4689         #endif
4690       }
4691     }
4692     else if(nop) {
4693       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4694       void *jaddr=out;
4695       emit_jns(0);
4696       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4697     }
4698     else {
4699       void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
4700       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
4701       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4702
4703       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4704       assert(s1l>=0);
4705       if(opcode[i]==4) // BEQ
4706       {
4707         if(s2l>=0) emit_cmp(s1l,s2l);
4708         else emit_test(s1l,s1l);
4709         if(invert){
4710           nottaken=out;
4711           emit_jne((void *)1l);
4712         }else{
4713           add_to_linker(out,ba[i],internal);
4714           emit_jeq(0);
4715         }
4716       }
4717       if(opcode[i]==5) // BNE
4718       {
4719         if(s2l>=0) emit_cmp(s1l,s2l);
4720         else emit_test(s1l,s1l);
4721         if(invert){
4722           nottaken=out;
4723           emit_jeq(1);
4724         }else{
4725           add_to_linker(out,ba[i],internal);
4726           emit_jne(0);
4727         }
4728       }
4729       if(opcode[i]==6) // BLEZ
4730       {
4731         emit_cmpimm(s1l,1);
4732         if(invert){
4733           nottaken=out;
4734           emit_jge(1);
4735         }else{
4736           add_to_linker(out,ba[i],internal);
4737           emit_jl(0);
4738         }
4739       }
4740       if(opcode[i]==7) // BGTZ
4741       {
4742         emit_cmpimm(s1l,1);
4743         if(invert){
4744           nottaken=out;
4745           emit_jl(1);
4746         }else{
4747           add_to_linker(out,ba[i],internal);
4748           emit_jge(0);
4749         }
4750       }
4751       if(invert) {
4752         if(taken) set_jump_target(taken, out);
4753         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4754         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
4755           if(adj) {
4756             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
4757             add_to_linker(out,ba[i],internal);
4758           }else{
4759             emit_addnop(13);
4760             add_to_linker(out,ba[i],internal*2);
4761           }
4762           emit_jmp(0);
4763         }else
4764         #endif
4765         {
4766           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
4767           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4768           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4769           if(internal)
4770             assem_debug("branch: internal\n");
4771           else
4772             assem_debug("branch: external\n");
4773           if(internal&&is_ds[(ba[i]-start)>>2]) {
4774             ds_assemble_entry(i);
4775           }
4776           else {
4777             add_to_linker(out,ba[i],internal);
4778             emit_jmp(0);
4779           }
4780         }
4781         set_jump_target(nottaken, out);
4782       }
4783
4784       if(nottaken1) set_jump_target(nottaken1, out);
4785       if(adj) {
4786         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
4787       }
4788     } // (!unconditional)
4789   } // if(ooo)
4790   else
4791   {
4792     // In-order execution (branch first)
4793     //if(likely[i]) printf("IOL\n");
4794     //else
4795     //printf("IOE\n");
4796     void *taken = NULL, *nottaken = NULL, *nottaken1 = NULL;
4797     if(!unconditional&&!nop) {
4798       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4799       assert(s1l>=0);
4800       if((opcode[i]&0x2f)==4) // BEQ
4801       {
4802         if(s2l>=0) emit_cmp(s1l,s2l);
4803         else emit_test(s1l,s1l);
4804         nottaken=out;
4805         emit_jne((void *)2l);
4806       }
4807       if((opcode[i]&0x2f)==5) // BNE
4808       {
4809         if(s2l>=0) emit_cmp(s1l,s2l);
4810         else emit_test(s1l,s1l);
4811         nottaken=out;
4812         emit_jeq(2);
4813       }
4814       if((opcode[i]&0x2f)==6) // BLEZ
4815       {
4816         emit_cmpimm(s1l,1);
4817         nottaken=out;
4818         emit_jge(2);
4819       }
4820       if((opcode[i]&0x2f)==7) // BGTZ
4821       {
4822         emit_cmpimm(s1l,1);
4823         nottaken=out;
4824         emit_jl(2);
4825       }
4826     } // if(!unconditional)
4827     int adj;
4828     uint64_t ds_unneeded=branch_regs[i].u;
4829     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
4830     ds_unneeded|=1;
4831     // branch taken
4832     if(!nop) {
4833       if(taken) set_jump_target(taken, out);
4834       assem_debug("1:\n");
4835       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
4836       // load regs
4837       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
4838       address_generation(i+1,&branch_regs[i],0);
4839       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
4840       ds_assemble(i+1,&branch_regs[i]);
4841       cc=get_reg(branch_regs[i].regmap,CCREG);
4842       if(cc==-1) {
4843         emit_loadreg(CCREG,cc=HOST_CCREG);
4844         // CHECK: Is the following instruction (fall thru) allocated ok?
4845       }
4846       assert(cc==HOST_CCREG);
4847       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4848       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
4849       assem_debug("cycle count (adj)\n");
4850       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4851       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4852       if(internal)
4853         assem_debug("branch: internal\n");
4854       else
4855         assem_debug("branch: external\n");
4856       if(internal&&is_ds[(ba[i]-start)>>2]) {
4857         ds_assemble_entry(i);
4858       }
4859       else {
4860         add_to_linker(out,ba[i],internal);
4861         emit_jmp(0);
4862       }
4863     }
4864     // branch not taken
4865     if(!unconditional) {
4866       if(nottaken1) set_jump_target(nottaken1, out);
4867       set_jump_target(nottaken, out);
4868       assem_debug("2:\n");
4869       if(!likely[i]) {
4870         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
4871         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
4872         address_generation(i+1,&branch_regs[i],0);
4873         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4874         ds_assemble(i+1,&branch_regs[i]);
4875       }
4876       cc=get_reg(branch_regs[i].regmap,CCREG);
4877       if(cc==-1&&!likely[i]) {
4878         // Cycle count isn't in a register, temporarily load it then write it out
4879         emit_loadreg(CCREG,HOST_CCREG);
4880         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
4881         void *jaddr=out;
4882         emit_jns(0);
4883         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4884         emit_storereg(CCREG,HOST_CCREG);
4885       }
4886       else{
4887         cc=get_reg(i_regmap,CCREG);
4888         assert(cc==HOST_CCREG);
4889         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4890         void *jaddr=out;
4891         emit_jns(0);
4892         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
4893       }
4894     }
4895   }
4896 }
4897
4898 void sjump_assemble(int i,struct regstat *i_regs)
4899 {
4900   signed char *i_regmap=i_regs->regmap;
4901   int cc;
4902   int match;
4903   match=match_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4904   assem_debug("smatch=%d\n",match);
4905   int s1l;
4906   int unconditional=0,nevertaken=0;
4907   int invert=0;
4908   int internal=internal_branch(ba[i]);
4909   if(i==(ba[i]-start)>>2) assem_debug("idle loop\n");
4910   if(!match) invert=1;
4911   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4912   if(i>(ba[i]-start)>>2) invert=1;
4913   #endif
4914
4915   //if(opcode2[i]>=0x10) return; // FIXME (BxxZAL)
4916   //assert(opcode2[i]<0x10||rs1[i]==0); // FIXME (BxxZAL)
4917
4918   if(ooo[i]) {
4919     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4920   }
4921   else {
4922     s1l=get_reg(i_regmap,rs1[i]);
4923   }
4924   if(rs1[i]==0)
4925   {
4926     if(opcode2[i]&1) unconditional=1;
4927     else nevertaken=1;
4928     // These are never taken (r0 is never less than zero)
4929     //assert(opcode2[i]!=0);
4930     //assert(opcode2[i]!=2);
4931     //assert(opcode2[i]!=0x10);
4932     //assert(opcode2[i]!=0x12);
4933   }
4934
4935   if(ooo[i]) {
4936     // Out of order execution (delay slot first)
4937     //printf("OOOE\n");
4938     address_generation(i+1,i_regs,regs[i].regmap_entry);
4939     ds_assemble(i+1,i_regs);
4940     int adj;
4941     uint64_t bc_unneeded=branch_regs[i].u;
4942     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
4943     bc_unneeded|=1;
4944     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,bc_unneeded);
4945     load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i],rs1[i]);
4946     load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
4947     if(rt1[i]==31) {
4948       int rt,return_address;
4949       rt=get_reg(branch_regs[i].regmap,31);
4950       assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
4951       if(rt>=0) {
4952         // Save the PC even if the branch is not taken
4953         return_address=start+i*4+8;
4954         emit_movimm(return_address,rt); // PC into link register
4955         #ifdef IMM_PREFETCH
4956         if(!nevertaken) emit_prefetch(hash_table_get(return_address));
4957         #endif
4958       }
4959     }
4960     cc=get_reg(branch_regs[i].regmap,CCREG);
4961     assert(cc==HOST_CCREG);
4962     if(unconditional)
4963       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4964     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
4965     assem_debug("cycle count (adj)\n");
4966     if(unconditional) {
4967       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
4968       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
4969         if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4970         load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
4971         if(internal)
4972           assem_debug("branch: internal\n");
4973         else
4974           assem_debug("branch: external\n");
4975         if(internal&&is_ds[(ba[i]-start)>>2]) {
4976           ds_assemble_entry(i);
4977         }
4978         else {
4979           add_to_linker(out,ba[i],internal);
4980           emit_jmp(0);
4981         }
4982         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
4983         if(((u_int)out)&7) emit_addnop(0);
4984         #endif
4985       }
4986     }
4987     else if(nevertaken) {
4988       emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
4989       void *jaddr=out;
4990       emit_jns(0);
4991       add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
4992     }
4993     else {
4994       void *nottaken = NULL;
4995       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
4996       if(adj&&!invert) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
4997       {
4998         assert(s1l>=0);
4999         if((opcode2[i]&0xf)==0) // BLTZ/BLTZAL
5000         {
5001           emit_test(s1l,s1l);
5002           if(invert){
5003             nottaken=out;
5004             emit_jns(1);
5005           }else{
5006             add_to_linker(out,ba[i],internal);
5007             emit_js(0);
5008           }
5009         }
5010         if((opcode2[i]&0xf)==1) // BGEZ/BLTZAL
5011         {
5012           emit_test(s1l,s1l);
5013           if(invert){
5014             nottaken=out;
5015             emit_js(1);
5016           }else{
5017             add_to_linker(out,ba[i],internal);
5018             emit_jns(0);
5019           }
5020         }
5021       }
5022
5023       if(invert) {
5024         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5025         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5026           if(adj) {
5027             emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5028             add_to_linker(out,ba[i],internal);
5029           }else{
5030             emit_addnop(13);
5031             add_to_linker(out,ba[i],internal*2);
5032           }
5033           emit_jmp(0);
5034         }else
5035         #endif
5036         {
5037           if(adj) emit_addimm(cc,-CLOCK_ADJUST(adj),cc);
5038           store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5039           load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5040           if(internal)
5041             assem_debug("branch: internal\n");
5042           else
5043             assem_debug("branch: external\n");
5044           if(internal&&is_ds[(ba[i]-start)>>2]) {
5045             ds_assemble_entry(i);
5046           }
5047           else {
5048             add_to_linker(out,ba[i],internal);
5049             emit_jmp(0);
5050           }
5051         }
5052         set_jump_target(nottaken, out);
5053       }
5054
5055       if(adj) {
5056         if(!invert) emit_addimm(cc,CLOCK_ADJUST(adj),cc);
5057       }
5058     } // (!unconditional)
5059   } // if(ooo)
5060   else
5061   {
5062     // In-order execution (branch first)
5063     //printf("IOE\n");
5064     void *nottaken = NULL;
5065     if(rt1[i]==31) {
5066       int rt,return_address;
5067       rt=get_reg(branch_regs[i].regmap,31);
5068       if(rt>=0) {
5069         // Save the PC even if the branch is not taken
5070         return_address=start+i*4+8;
5071         emit_movimm(return_address,rt); // PC into link register
5072         #ifdef IMM_PREFETCH
5073         emit_prefetch(hash_table_get(return_address));
5074         #endif
5075       }
5076     }
5077     if(!unconditional) {
5078       //printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5079         assert(s1l>=0);
5080         if((opcode2[i]&0x0d)==0) // BLTZ/BLTZL/BLTZAL/BLTZALL
5081         {
5082           emit_test(s1l,s1l);
5083           nottaken=out;
5084           emit_jns(1);
5085         }
5086         if((opcode2[i]&0x0d)==1) // BGEZ/BGEZL/BGEZAL/BGEZALL
5087         {
5088           emit_test(s1l,s1l);
5089           nottaken=out;
5090           emit_js(1);
5091         }
5092     } // if(!unconditional)
5093     int adj;
5094     uint64_t ds_unneeded=branch_regs[i].u;
5095     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5096     ds_unneeded|=1;
5097     // branch taken
5098     if(!nevertaken) {
5099       //assem_debug("1:\n");
5100       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5101       // load regs
5102       load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5103       address_generation(i+1,&branch_regs[i],0);
5104       load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,INVCP);
5105       ds_assemble(i+1,&branch_regs[i]);
5106       cc=get_reg(branch_regs[i].regmap,CCREG);
5107       if(cc==-1) {
5108         emit_loadreg(CCREG,cc=HOST_CCREG);
5109         // CHECK: Is the following instruction (fall thru) allocated ok?
5110       }
5111       assert(cc==HOST_CCREG);
5112       store_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5113       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5114       assem_debug("cycle count (adj)\n");
5115       if(adj) emit_addimm(cc,CLOCK_ADJUST(ccadj[i]+2-adj),cc);
5116       load_regs_bt(branch_regs[i].regmap,branch_regs[i].dirty,ba[i]);
5117       if(internal)
5118         assem_debug("branch: internal\n");
5119       else
5120         assem_debug("branch: external\n");
5121       if(internal&&is_ds[(ba[i]-start)>>2]) {
5122         ds_assemble_entry(i);
5123       }
5124       else {
5125         add_to_linker(out,ba[i],internal);
5126         emit_jmp(0);
5127       }
5128     }
5129     // branch not taken
5130     if(!unconditional) {
5131       set_jump_target(nottaken, out);
5132       assem_debug("1:\n");
5133       if(!likely[i]) {
5134         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,ds_unneeded);
5135         load_regs(regs[i].regmap,branch_regs[i].regmap,rs1[i+1],rs2[i+1]);
5136         address_generation(i+1,&branch_regs[i],0);
5137         load_regs(regs[i].regmap,branch_regs[i].regmap,CCREG,CCREG);
5138         ds_assemble(i+1,&branch_regs[i]);
5139       }
5140       cc=get_reg(branch_regs[i].regmap,CCREG);
5141       if(cc==-1&&!likely[i]) {
5142         // Cycle count isn't in a register, temporarily load it then write it out
5143         emit_loadreg(CCREG,HOST_CCREG);
5144         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5145         void *jaddr=out;
5146         emit_jns(0);
5147         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,NOTTAKEN,0);
5148         emit_storereg(CCREG,HOST_CCREG);
5149       }
5150       else{
5151         cc=get_reg(i_regmap,CCREG);
5152         assert(cc==HOST_CCREG);
5153         emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),cc);
5154         void *jaddr=out;
5155         emit_jns(0);
5156         add_stub(CC_STUB,jaddr,out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5157       }
5158     }
5159   }
5160 }
5161
5162 static void pagespan_assemble(int i,struct regstat *i_regs)
5163 {
5164   int s1l=get_reg(i_regs->regmap,rs1[i]);
5165   int s2l=get_reg(i_regs->regmap,rs2[i]);
5166   void *taken = NULL;
5167   void *nottaken = NULL;
5168   int unconditional=0;
5169   if(rs1[i]==0)
5170   {
5171     s1l=s2l;
5172     s2l=-1;
5173   }
5174   else if(rs2[i]==0)
5175   {
5176     s2l=-1;
5177   }
5178   int hr=0;
5179   int addr=-1,alt=-1,ntaddr=-1;
5180   if(i_regs->regmap[HOST_BTREG]<0) {addr=HOST_BTREG;}
5181   else {
5182     while(hr<HOST_REGS)
5183     {
5184       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
5185          (i_regs->regmap[hr]&63)!=rs1[i] &&
5186          (i_regs->regmap[hr]&63)!=rs2[i] )
5187       {
5188         addr=hr++;break;
5189       }
5190       hr++;
5191     }
5192   }
5193   while(hr<HOST_REGS)
5194   {
5195     if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5196        (i_regs->regmap[hr]&63)!=rs1[i] &&
5197        (i_regs->regmap[hr]&63)!=rs2[i] )
5198     {
5199       alt=hr++;break;
5200     }
5201     hr++;
5202   }
5203   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
5204   {
5205     while(hr<HOST_REGS)
5206     {
5207       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
5208          (i_regs->regmap[hr]&63)!=rs1[i] &&
5209          (i_regs->regmap[hr]&63)!=rs2[i] )
5210       {
5211         ntaddr=hr;break;
5212       }
5213       hr++;
5214     }
5215   }
5216   assert(hr<HOST_REGS);
5217   if((opcode[i]&0x2e)==4||opcode[i]==0x11) { // BEQ/BNE/BEQL/BNEL/BC1
5218     load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
5219   }
5220   emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG);
5221   if(opcode[i]==2) // J
5222   {
5223     unconditional=1;
5224   }
5225   if(opcode[i]==3) // JAL
5226   {
5227     // TODO: mini_ht
5228     int rt=get_reg(i_regs->regmap,31);
5229     emit_movimm(start+i*4+8,rt);
5230     unconditional=1;
5231   }
5232   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
5233   {
5234     emit_mov(s1l,addr);
5235     if(opcode2[i]==9) // JALR
5236     {
5237       int rt=get_reg(i_regs->regmap,rt1[i]);
5238       emit_movimm(start+i*4+8,rt);
5239     }
5240   }
5241   if((opcode[i]&0x3f)==4) // BEQ
5242   {
5243     if(rs1[i]==rs2[i])
5244     {
5245       unconditional=1;
5246     }
5247     else
5248     #ifdef HAVE_CMOV_IMM
5249     if(1) {
5250       if(s2l>=0) emit_cmp(s1l,s2l);
5251       else emit_test(s1l,s1l);
5252       emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
5253     }
5254     else
5255     #endif
5256     {
5257       assert(s1l>=0);
5258       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5259       if(s2l>=0) emit_cmp(s1l,s2l);
5260       else emit_test(s1l,s1l);
5261       emit_cmovne_reg(alt,addr);
5262     }
5263   }
5264   if((opcode[i]&0x3f)==5) // BNE
5265   {
5266     #ifdef HAVE_CMOV_IMM
5267     if(s2l>=0) emit_cmp(s1l,s2l);
5268     else emit_test(s1l,s1l);
5269     emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
5270     #else
5271     assert(s1l>=0);
5272     emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
5273     if(s2l>=0) emit_cmp(s1l,s2l);
5274     else emit_test(s1l,s1l);
5275     emit_cmovne_reg(alt,addr);
5276     #endif
5277   }
5278   if((opcode[i]&0x3f)==0x14) // BEQL
5279   {
5280     if(s2l>=0) emit_cmp(s1l,s2l);
5281     else emit_test(s1l,s1l);
5282     if(nottaken) set_jump_target(nottaken, out);
5283     nottaken=out;
5284     emit_jne(0);
5285   }
5286   if((opcode[i]&0x3f)==0x15) // BNEL
5287   {
5288     if(s2l>=0) emit_cmp(s1l,s2l);
5289     else emit_test(s1l,s1l);
5290     nottaken=out;
5291     emit_jeq(0);
5292     if(taken) set_jump_target(taken, out);
5293   }
5294   if((opcode[i]&0x3f)==6) // BLEZ
5295   {
5296     emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5297     emit_cmpimm(s1l,1);
5298     emit_cmovl_reg(alt,addr);
5299   }
5300   if((opcode[i]&0x3f)==7) // BGTZ
5301   {
5302     emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
5303     emit_cmpimm(s1l,1);
5304     emit_cmovl_reg(ntaddr,addr);
5305   }
5306   if((opcode[i]&0x3f)==0x16) // BLEZL
5307   {
5308     assert((opcode[i]&0x3f)!=0x16);
5309   }
5310   if((opcode[i]&0x3f)==0x17) // BGTZL
5311   {
5312     assert((opcode[i]&0x3f)!=0x17);
5313   }
5314   assert(opcode[i]!=1); // BLTZ/BGEZ
5315
5316   //FIXME: Check CSREG
5317   if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
5318     if((source[i]&0x30000)==0) // BC1F
5319     {
5320       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
5321       emit_testimm(s1l,0x800000);
5322       emit_cmovne_reg(alt,addr);
5323     }
5324     if((source[i]&0x30000)==0x10000) // BC1T
5325     {
5326       emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
5327       emit_testimm(s1l,0x800000);
5328       emit_cmovne_reg(alt,addr);
5329     }
5330     if((source[i]&0x30000)==0x20000) // BC1FL
5331     {
5332       emit_testimm(s1l,0x800000);
5333       nottaken=out;
5334       emit_jne(0);
5335     }
5336     if((source[i]&0x30000)==0x30000) // BC1TL
5337     {
5338       emit_testimm(s1l,0x800000);
5339       nottaken=out;
5340       emit_jeq(0);
5341     }
5342   }
5343
5344   assert(i_regs->regmap[HOST_CCREG]==CCREG);
5345   wb_dirtys(regs[i].regmap,regs[i].dirty);
5346   if(likely[i]||unconditional)
5347   {
5348     emit_movimm(ba[i],HOST_BTREG);
5349   }
5350   else if(addr!=HOST_BTREG)
5351   {
5352     emit_mov(addr,HOST_BTREG);
5353   }
5354   void *branch_addr=out;
5355   emit_jmp(0);
5356   int target_addr=start+i*4+5;
5357   void *stub=out;
5358   void *compiled_target_addr=check_addr(target_addr);
5359   emit_extjump_ds(branch_addr, target_addr);
5360   if(compiled_target_addr) {
5361     set_jump_target(branch_addr, compiled_target_addr);
5362     add_link(target_addr,stub);
5363   }
5364   else set_jump_target(branch_addr, stub);
5365   if(likely[i]) {
5366     // Not-taken path
5367     set_jump_target(nottaken, out);
5368     wb_dirtys(regs[i].regmap,regs[i].dirty);
5369     void *branch_addr=out;
5370     emit_jmp(0);
5371     int target_addr=start+i*4+8;
5372     void *stub=out;
5373     void *compiled_target_addr=check_addr(target_addr);
5374     emit_extjump_ds(branch_addr, target_addr);
5375     if(compiled_target_addr) {
5376       set_jump_target(branch_addr, compiled_target_addr);
5377       add_link(target_addr,stub);
5378     }
5379     else set_jump_target(branch_addr, stub);
5380   }
5381 }
5382
5383 // Assemble the delay slot for the above
5384 static void pagespan_ds()
5385 {
5386   assem_debug("initial delay slot:\n");
5387   u_int vaddr=start+1;
5388   u_int page=get_page(vaddr);
5389   u_int vpage=get_vpage(vaddr);
5390   ll_add(jump_dirty+vpage,vaddr,(void *)out);
5391   do_dirty_stub_ds();
5392   ll_add(jump_in+page,vaddr,(void *)out);
5393   assert(regs[0].regmap_entry[HOST_CCREG]==CCREG);
5394   if(regs[0].regmap[HOST_CCREG]!=CCREG)
5395     wb_register(CCREG,regs[0].regmap_entry,regs[0].wasdirty);
5396   if(regs[0].regmap[HOST_BTREG]!=BTREG)
5397     emit_writeword(HOST_BTREG,&branch_target);
5398   load_regs(regs[0].regmap_entry,regs[0].regmap,rs1[0],rs2[0]);
5399   address_generation(0,&regs[0],regs[0].regmap_entry);
5400   if(itype[0]==STORE||itype[0]==STORELR||(opcode[0]&0x3b)==0x39||(opcode[0]&0x3b)==0x3a)
5401     load_regs(regs[0].regmap_entry,regs[0].regmap,INVCP,INVCP);
5402   is_delayslot=0;
5403   switch(itype[0]) {
5404     case ALU:
5405       alu_assemble(0,&regs[0]);break;
5406     case IMM16:
5407       imm16_assemble(0,&regs[0]);break;
5408     case SHIFT:
5409       shift_assemble(0,&regs[0]);break;
5410     case SHIFTIMM:
5411       shiftimm_assemble(0,&regs[0]);break;
5412     case LOAD:
5413       load_assemble(0,&regs[0]);break;
5414     case LOADLR:
5415       loadlr_assemble(0,&regs[0]);break;
5416     case STORE:
5417       store_assemble(0,&regs[0]);break;
5418     case STORELR:
5419       storelr_assemble(0,&regs[0]);break;
5420     case COP0:
5421       cop0_assemble(0,&regs[0]);break;
5422     case COP1:
5423       cop1_assemble(0,&regs[0]);break;
5424     case C1LS:
5425       c1ls_assemble(0,&regs[0]);break;
5426     case COP2:
5427       cop2_assemble(0,&regs[0]);break;
5428     case C2LS:
5429       c2ls_assemble(0,&regs[0]);break;
5430     case C2OP:
5431       c2op_assemble(0,&regs[0]);break;
5432     case MULTDIV:
5433       multdiv_assemble(0,&regs[0]);break;
5434     case MOV:
5435       mov_assemble(0,&regs[0]);break;
5436     case SYSCALL:
5437     case HLECALL:
5438     case INTCALL:
5439     case SPAN:
5440     case UJUMP:
5441     case RJUMP:
5442     case CJUMP:
5443     case SJUMP:
5444       SysPrintf("Jump in the delay slot.  This is probably a bug.\n");
5445   }
5446   int btaddr=get_reg(regs[0].regmap,BTREG);
5447   if(btaddr<0) {
5448     btaddr=get_reg(regs[0].regmap,-1);
5449     emit_readword(&branch_target,btaddr);
5450   }
5451   assert(btaddr!=HOST_CCREG);
5452   if(regs[0].regmap[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
5453 #ifdef HOST_IMM8
5454   emit_movimm(start+4,HOST_TEMPREG);
5455   emit_cmp(btaddr,HOST_TEMPREG);
5456 #else
5457   emit_cmpimm(btaddr,start+4);
5458 #endif
5459   void *branch = out;
5460   emit_jeq(0);
5461   store_regs_bt(regs[0].regmap,regs[0].dirty,-1);
5462   emit_jmp(jump_vaddr_reg[btaddr]);
5463   set_jump_target(branch, out);
5464   store_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5465   load_regs_bt(regs[0].regmap,regs[0].dirty,start+4);
5466 }
5467
5468 // Basic liveness analysis for MIPS registers
5469 void unneeded_registers(int istart,int iend,int r)
5470 {
5471   int i;
5472   uint64_t u,gte_u,b,gte_b;
5473   uint64_t temp_u,temp_gte_u=0;
5474   uint64_t gte_u_unknown=0;
5475   if(new_dynarec_hacks&NDHACK_GTE_UNNEEDED)
5476     gte_u_unknown=~0ll;
5477   if(iend==slen-1) {
5478     u=1;
5479     gte_u=gte_u_unknown;
5480   }else{
5481     //u=unneeded_reg[iend+1];
5482     u=1;
5483     gte_u=gte_unneeded[iend+1];
5484   }
5485
5486   for (i=iend;i>=istart;i--)
5487   {
5488     //printf("unneeded registers i=%d (%d,%d) r=%d\n",i,istart,iend,r);
5489     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5490     {
5491       // If subroutine call, flag return address as a possible branch target
5492       if(rt1[i]==31 && i<slen-2) bt[i+2]=1;
5493
5494       if(ba[i]<start || ba[i]>=(start+slen*4))
5495       {
5496         // Branch out of this block, flush all regs
5497         u=1;
5498         gte_u=gte_u_unknown;
5499         branch_unneeded_reg[i]=u;
5500         // Merge in delay slot
5501         u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5502         u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5503         u|=1;
5504         gte_u|=gte_rt[i+1];
5505         gte_u&=~gte_rs[i+1];
5506         // If branch is "likely" (and conditional)
5507         // then we skip the delay slot on the fall-thru path
5508         if(likely[i]) {
5509           if(i<slen-1) {
5510             u&=unneeded_reg[i+2];
5511             gte_u&=gte_unneeded[i+2];
5512           }
5513           else
5514           {
5515             u=1;
5516             gte_u=gte_u_unknown;
5517           }
5518         }
5519       }
5520       else
5521       {
5522         // Internal branch, flag target
5523         bt[(ba[i]-start)>>2]=1;
5524         if(ba[i]<=start+i*4) {
5525           // Backward branch
5526           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5527           {
5528             // Unconditional branch
5529             temp_u=1;
5530             temp_gte_u=0;
5531           } else {
5532             // Conditional branch (not taken case)
5533             temp_u=unneeded_reg[i+2];
5534             temp_gte_u&=gte_unneeded[i+2];
5535           }
5536           // Merge in delay slot
5537           temp_u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5538           temp_u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5539           temp_u|=1;
5540           temp_gte_u|=gte_rt[i+1];
5541           temp_gte_u&=~gte_rs[i+1];
5542           // If branch is "likely" (and conditional)
5543           // then we skip the delay slot on the fall-thru path
5544           if(likely[i]) {
5545             if(i<slen-1) {
5546               temp_u&=unneeded_reg[i+2];
5547               temp_gte_u&=gte_unneeded[i+2];
5548             }
5549             else
5550             {
5551               temp_u=1;
5552               temp_gte_u=gte_u_unknown;
5553             }
5554           }
5555           temp_u|=(1LL<<rt1[i])|(1LL<<rt2[i]);
5556           temp_u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5557           temp_u|=1;
5558           temp_gte_u|=gte_rt[i];
5559           temp_gte_u&=~gte_rs[i];
5560           unneeded_reg[i]=temp_u;
5561           gte_unneeded[i]=temp_gte_u;
5562           // Only go three levels deep.  This recursion can take an
5563           // excessive amount of time if there are a lot of nested loops.
5564           if(r<2) {
5565             unneeded_registers((ba[i]-start)>>2,i-1,r+1);
5566           }else{
5567             unneeded_reg[(ba[i]-start)>>2]=1;
5568             gte_unneeded[(ba[i]-start)>>2]=gte_u_unknown;
5569           }
5570         } /*else*/ if(1) {
5571           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5572           {
5573             // Unconditional branch
5574             u=unneeded_reg[(ba[i]-start)>>2];
5575             gte_u=gte_unneeded[(ba[i]-start)>>2];
5576             branch_unneeded_reg[i]=u;
5577             // Merge in delay slot
5578             u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5579             u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5580             u|=1;
5581             gte_u|=gte_rt[i+1];
5582             gte_u&=~gte_rs[i+1];
5583           } else {
5584             // Conditional branch
5585             b=unneeded_reg[(ba[i]-start)>>2];
5586             gte_b=gte_unneeded[(ba[i]-start)>>2];
5587             branch_unneeded_reg[i]=b;
5588             // Branch delay slot
5589             b|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
5590             b&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5591             b|=1;
5592             gte_b|=gte_rt[i+1];
5593             gte_b&=~gte_rs[i+1];
5594             // If branch is "likely" then we skip the
5595             // delay slot on the fall-thru path
5596             if(likely[i]) {
5597               u=b;
5598               gte_u=gte_b;
5599               if(i<slen-1) {
5600                 u&=unneeded_reg[i+2];
5601                 gte_u&=gte_unneeded[i+2];
5602               }
5603             } else {
5604               u&=b;
5605               gte_u&=gte_b;
5606             }
5607             if(i<slen-1) {
5608               branch_unneeded_reg[i]&=unneeded_reg[i+2];
5609             } else {
5610               branch_unneeded_reg[i]=1;
5611             }
5612           }
5613         }
5614       }
5615     }
5616     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5617     {
5618       // SYSCALL instruction (software interrupt)
5619       u=1;
5620     }
5621     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5622     {
5623       // ERET instruction (return from interrupt)
5624       u=1;
5625     }
5626     //u=1; // DEBUG
5627     // Written registers are unneeded
5628     u|=1LL<<rt1[i];
5629     u|=1LL<<rt2[i];
5630     gte_u|=gte_rt[i];
5631     // Accessed registers are needed
5632     u&=~(1LL<<rs1[i]);
5633     u&=~(1LL<<rs2[i]);
5634     gte_u&=~gte_rs[i];
5635     if(gte_rs[i]&&rt1[i]&&(unneeded_reg[i+1]&(1ll<<rt1[i])))
5636       gte_u|=gte_rs[i]&gte_unneeded[i+1]; // MFC2/CFC2 to dead register, unneeded
5637     // Source-target dependencies
5638     // R0 is always unneeded
5639     u|=1;
5640     // Save it
5641     unneeded_reg[i]=u;
5642     gte_unneeded[i]=gte_u;
5643     /*
5644     printf("ur (%d,%d) %x: ",istart,iend,start+i*4);
5645     printf("U:");
5646     int r;
5647     for(r=1;r<=CCREG;r++) {
5648       if((unneeded_reg[i]>>r)&1) {
5649         if(r==HIREG) printf(" HI");
5650         else if(r==LOREG) printf(" LO");
5651         else printf(" r%d",r);
5652       }
5653     }
5654     printf("\n");
5655     */
5656   }
5657 }
5658
5659 // Write back dirty registers as soon as we will no longer modify them,
5660 // so that we don't end up with lots of writes at the branches.
5661 void clean_registers(int istart,int iend,int wr)
5662 {
5663   int i;
5664   int r;
5665   u_int will_dirty_i,will_dirty_next,temp_will_dirty;
5666   u_int wont_dirty_i,wont_dirty_next,temp_wont_dirty;
5667   if(iend==slen-1) {
5668     will_dirty_i=will_dirty_next=0;
5669     wont_dirty_i=wont_dirty_next=0;
5670   }else{
5671     will_dirty_i=will_dirty_next=will_dirty[iend+1];
5672     wont_dirty_i=wont_dirty_next=wont_dirty[iend+1];
5673   }
5674   for (i=iend;i>=istart;i--)
5675   {
5676     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
5677     {
5678       if(ba[i]<start || ba[i]>=(start+slen*4))
5679       {
5680         // Branch out of this block, flush all regs
5681         if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5682         {
5683           // Unconditional branch
5684           will_dirty_i=0;
5685           wont_dirty_i=0;
5686           // Merge in delay slot (will dirty)
5687           for(r=0;r<HOST_REGS;r++) {
5688             if(r!=EXCLUDE_REG) {
5689               if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5690               if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5691               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5692               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5693               if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5694               if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5695               if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5696               if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5697               if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5698               if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5699               if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5700               if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5701               if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5702               if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5703             }
5704           }
5705         }
5706         else
5707         {
5708           // Conditional branch
5709           will_dirty_i=0;
5710           wont_dirty_i=wont_dirty_next;
5711           // Merge in delay slot (will dirty)
5712           for(r=0;r<HOST_REGS;r++) {
5713             if(r!=EXCLUDE_REG) {
5714               if(!likely[i]) {
5715                 // Might not dirty if likely branch is not taken
5716                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5717                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5718                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5719                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5720                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5721                 if(branch_regs[i].regmap[r]==0) will_dirty_i&=~(1<<r);
5722                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5723                 //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5724                 //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5725                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5726                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5727                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5728                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5729                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5730               }
5731             }
5732           }
5733         }
5734         // Merge in delay slot (wont dirty)
5735         for(r=0;r<HOST_REGS;r++) {
5736           if(r!=EXCLUDE_REG) {
5737             if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5738             if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5739             if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5740             if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5741             if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5742             if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5743             if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5744             if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5745             if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5746             if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5747           }
5748         }
5749         if(wr) {
5750           #ifndef DESTRUCTIVE_WRITEBACK
5751           branch_regs[i].dirty&=wont_dirty_i;
5752           #endif
5753           branch_regs[i].dirty|=will_dirty_i;
5754         }
5755       }
5756       else
5757       {
5758         // Internal branch
5759         if(ba[i]<=start+i*4) {
5760           // Backward branch
5761           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5762           {
5763             // Unconditional branch
5764             temp_will_dirty=0;
5765             temp_wont_dirty=0;
5766             // Merge in delay slot (will dirty)
5767             for(r=0;r<HOST_REGS;r++) {
5768               if(r!=EXCLUDE_REG) {
5769                 if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5770                 if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5771                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5772                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5773                 if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5774                 if(branch_regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5775                 if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5776                 if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5777                 if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5778                 if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5779                 if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5780                 if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5781                 if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5782                 if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5783               }
5784             }
5785           } else {
5786             // Conditional branch (not taken case)
5787             temp_will_dirty=will_dirty_next;
5788             temp_wont_dirty=wont_dirty_next;
5789             // Merge in delay slot (will dirty)
5790             for(r=0;r<HOST_REGS;r++) {
5791               if(r!=EXCLUDE_REG) {
5792                 if(!likely[i]) {
5793                   // Will not dirty if likely branch is not taken
5794                   if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5795                   if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5796                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5797                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5798                   if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5799                   if(branch_regs[i].regmap[r]==0) temp_will_dirty&=~(1<<r);
5800                   if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5801                   //if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
5802                   //if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
5803                   if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
5804                   if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
5805                   if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
5806                   if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
5807                   if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
5808                 }
5809               }
5810             }
5811           }
5812           // Merge in delay slot (wont dirty)
5813           for(r=0;r<HOST_REGS;r++) {
5814             if(r!=EXCLUDE_REG) {
5815               if((regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
5816               if((regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
5817               if((regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
5818               if((regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
5819               if(regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
5820               if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
5821               if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
5822               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
5823               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
5824               if(branch_regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
5825             }
5826           }
5827           // Deal with changed mappings
5828           if(i<iend) {
5829             for(r=0;r<HOST_REGS;r++) {
5830               if(r!=EXCLUDE_REG) {
5831                 if(regs[i].regmap[r]!=regmap_pre[i][r]) {
5832                   temp_will_dirty&=~(1<<r);
5833                   temp_wont_dirty&=~(1<<r);
5834                   if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
5835                     temp_will_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
5836                     temp_wont_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
5837                   } else {
5838                     temp_will_dirty|=1<<r;
5839                     temp_wont_dirty|=1<<r;
5840                   }
5841                 }
5842               }
5843             }
5844           }
5845           if(wr) {
5846             will_dirty[i]=temp_will_dirty;
5847             wont_dirty[i]=temp_wont_dirty;
5848             clean_registers((ba[i]-start)>>2,i-1,0);
5849           }else{
5850             // Limit recursion.  It can take an excessive amount
5851             // of time if there are a lot of nested loops.
5852             will_dirty[(ba[i]-start)>>2]=0;
5853             wont_dirty[(ba[i]-start)>>2]=-1;
5854           }
5855         }
5856         /*else*/ if(1)
5857         {
5858           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
5859           {
5860             // Unconditional branch
5861             will_dirty_i=0;
5862             wont_dirty_i=0;
5863           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
5864             for(r=0;r<HOST_REGS;r++) {
5865               if(r!=EXCLUDE_REG) {
5866                 if(branch_regs[i].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5867                   will_dirty_i|=will_dirty[(ba[i]-start)>>2]&(1<<r);
5868                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5869                 }
5870                 if(branch_regs[i].regmap[r]>=0) {
5871                   will_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
5872                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
5873                 }
5874               }
5875             }
5876           //}
5877             // Merge in delay slot
5878             for(r=0;r<HOST_REGS;r++) {
5879               if(r!=EXCLUDE_REG) {
5880                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5881                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5882                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5883                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5884                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5885                 if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5886                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5887                 if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5888                 if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5889                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5890                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5891                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5892                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5893                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5894               }
5895             }
5896           } else {
5897             // Conditional branch
5898             will_dirty_i=will_dirty_next;
5899             wont_dirty_i=wont_dirty_next;
5900           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
5901             for(r=0;r<HOST_REGS;r++) {
5902               if(r!=EXCLUDE_REG) {
5903                 signed char target_reg=branch_regs[i].regmap[r];
5904                 if(target_reg==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5905                   will_dirty_i&=will_dirty[(ba[i]-start)>>2]&(1<<r);
5906                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5907                 }
5908                 else if(target_reg>=0) {
5909                   will_dirty_i&=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
5910                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
5911                 }
5912                 // Treat delay slot as part of branch too
5913                 /*if(regs[i+1].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
5914                   will_dirty[i+1]&=will_dirty[(ba[i]-start)>>2]&(1<<r);
5915                   wont_dirty[i+1]|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
5916                 }
5917                 else
5918                 {
5919                   will_dirty[i+1]&=~(1<<r);
5920                 }*/
5921               }
5922             }
5923           //}
5924             // Merge in delay slot
5925             for(r=0;r<HOST_REGS;r++) {
5926               if(r!=EXCLUDE_REG) {
5927                 if(!likely[i]) {
5928                   // Might not dirty if likely branch is not taken
5929                   if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5930                   if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5931                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5932                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5933                   if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5934                   if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5935                   if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5936                   //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5937                   //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5938                   if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
5939                   if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
5940                   if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5941                   if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5942                   if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5943                 }
5944               }
5945             }
5946           }
5947           // Merge in delay slot (won't dirty)
5948           for(r=0;r<HOST_REGS;r++) {
5949             if(r!=EXCLUDE_REG) {
5950               if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5951               if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5952               if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5953               if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5954               if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5955               if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5956               if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5957               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
5958               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
5959               if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5960             }
5961           }
5962           if(wr) {
5963             #ifndef DESTRUCTIVE_WRITEBACK
5964             branch_regs[i].dirty&=wont_dirty_i;
5965             #endif
5966             branch_regs[i].dirty|=will_dirty_i;
5967           }
5968         }
5969       }
5970     }
5971     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
5972     {
5973       // SYSCALL instruction (software interrupt)
5974       will_dirty_i=0;
5975       wont_dirty_i=0;
5976     }
5977     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
5978     {
5979       // ERET instruction (return from interrupt)
5980       will_dirty_i=0;
5981       wont_dirty_i=0;
5982     }
5983     will_dirty_next=will_dirty_i;
5984     wont_dirty_next=wont_dirty_i;
5985     for(r=0;r<HOST_REGS;r++) {
5986       if(r!=EXCLUDE_REG) {
5987         if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
5988         if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
5989         if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
5990         if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
5991         if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
5992         if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
5993         if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
5994         if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
5995         if(i>istart) {
5996           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP)
5997           {
5998             // Don't store a register immediately after writing it,
5999             // may prevent dual-issue.
6000             if((regs[i].regmap[r]&63)==rt1[i-1]) wont_dirty_i|=1<<r;
6001             if((regs[i].regmap[r]&63)==rt2[i-1]) wont_dirty_i|=1<<r;
6002           }
6003         }
6004       }
6005     }
6006     // Save it
6007     will_dirty[i]=will_dirty_i;
6008     wont_dirty[i]=wont_dirty_i;
6009     // Mark registers that won't be dirtied as not dirty
6010     if(wr) {
6011       /*printf("wr (%d,%d) %x will:",istart,iend,start+i*4);
6012       for(r=0;r<HOST_REGS;r++) {
6013         if((will_dirty_i>>r)&1) {
6014           printf(" r%d",r);
6015         }
6016       }
6017       printf("\n");*/
6018
6019       //if(i==istart||(itype[i-1]!=RJUMP&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP)) {
6020         regs[i].dirty|=will_dirty_i;
6021         #ifndef DESTRUCTIVE_WRITEBACK
6022         regs[i].dirty&=wont_dirty_i;
6023         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
6024         {
6025           if(i<iend-1&&itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
6026             for(r=0;r<HOST_REGS;r++) {
6027               if(r!=EXCLUDE_REG) {
6028                 if(regs[i].regmap[r]==regmap_pre[i+2][r]) {
6029                   regs[i+2].wasdirty&=wont_dirty_i|~(1<<r);
6030                 }else {/*printf("i: %x (%d) mismatch(+2): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6031               }
6032             }
6033           }
6034         }
6035         else
6036         {
6037           if(i<iend) {
6038             for(r=0;r<HOST_REGS;r++) {
6039               if(r!=EXCLUDE_REG) {
6040                 if(regs[i].regmap[r]==regmap_pre[i+1][r]) {
6041                   regs[i+1].wasdirty&=wont_dirty_i|~(1<<r);
6042                 }else {/*printf("i: %x (%d) mismatch(+1): %d\n",start+i*4,i,r);assert(!((wont_dirty_i>>r)&1));*/}
6043               }
6044             }
6045           }
6046         }
6047         #endif
6048       //}
6049     }
6050     // Deal with changed mappings
6051     temp_will_dirty=will_dirty_i;
6052     temp_wont_dirty=wont_dirty_i;
6053     for(r=0;r<HOST_REGS;r++) {
6054       if(r!=EXCLUDE_REG) {
6055         int nr;
6056         if(regs[i].regmap[r]==regmap_pre[i][r]) {
6057           if(wr) {
6058             #ifndef DESTRUCTIVE_WRITEBACK
6059             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6060             #endif
6061             regs[i].wasdirty|=will_dirty_i&(1<<r);
6062           }
6063         }
6064         else if(regmap_pre[i][r]>=0&&(nr=get_reg(regs[i].regmap,regmap_pre[i][r]))>=0) {
6065           // Register moved to a different register
6066           will_dirty_i&=~(1<<r);
6067           wont_dirty_i&=~(1<<r);
6068           will_dirty_i|=((temp_will_dirty>>nr)&1)<<r;
6069           wont_dirty_i|=((temp_wont_dirty>>nr)&1)<<r;
6070           if(wr) {
6071             #ifndef DESTRUCTIVE_WRITEBACK
6072             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
6073             #endif
6074             regs[i].wasdirty|=will_dirty_i&(1<<r);
6075           }
6076         }
6077         else {
6078           will_dirty_i&=~(1<<r);
6079           wont_dirty_i&=~(1<<r);
6080           if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
6081             will_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6082             wont_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
6083           } else {
6084             wont_dirty_i|=1<<r;
6085             /*printf("i: %x (%d) mismatch: %d\n",start+i*4,i,r);assert(!((will_dirty>>r)&1));*/
6086           }
6087         }
6088       }
6089     }
6090   }
6091 }
6092
6093 #ifdef DISASM
6094   /* disassembly */
6095 void disassemble_inst(int i)
6096 {
6097     if (bt[i]) printf("*"); else printf(" ");
6098     switch(itype[i]) {
6099       case UJUMP:
6100         printf (" %x: %s %8x\n",start+i*4,insn[i],ba[i]);break;
6101       case CJUMP:
6102         printf (" %x: %s r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],i?start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14):*ba);break;
6103       case SJUMP:
6104         printf (" %x: %s r%d,%8x\n",start+i*4,insn[i],rs1[i],start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14));break;
6105       case RJUMP:
6106         if (opcode[i]==0x9&&rt1[i]!=31)
6107           printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i]);
6108         else
6109           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6110         break;
6111       case SPAN:
6112         printf (" %x: %s (pagespan) r%d,r%d,%8x\n",start+i*4,insn[i],rs1[i],rs2[i],ba[i]);break;
6113       case IMM16:
6114         if(opcode[i]==0xf) //LUI
6115           printf (" %x: %s r%d,%4x0000\n",start+i*4,insn[i],rt1[i],imm[i]&0xffff);
6116         else
6117           printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6118         break;
6119       case LOAD:
6120       case LOADLR:
6121         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6122         break;
6123       case STORE:
6124       case STORELR:
6125         printf (" %x: %s r%d,r%d+%x\n",start+i*4,insn[i],rs2[i],rs1[i],imm[i]);
6126         break;
6127       case ALU:
6128       case SHIFT:
6129         printf (" %x: %s r%d,r%d,r%d\n",start+i*4,insn[i],rt1[i],rs1[i],rs2[i]);
6130         break;
6131       case MULTDIV:
6132         printf (" %x: %s r%d,r%d\n",start+i*4,insn[i],rs1[i],rs2[i]);
6133         break;
6134       case SHIFTIMM:
6135         printf (" %x: %s r%d,r%d,%d\n",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
6136         break;
6137       case MOV:
6138         if((opcode2[i]&0x1d)==0x10)
6139           printf (" %x: %s r%d\n",start+i*4,insn[i],rt1[i]);
6140         else if((opcode2[i]&0x1d)==0x11)
6141           printf (" %x: %s r%d\n",start+i*4,insn[i],rs1[i]);
6142         else
6143           printf (" %x: %s\n",start+i*4,insn[i]);
6144         break;
6145       case COP0:
6146         if(opcode2[i]==0)
6147           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC0
6148         else if(opcode2[i]==4)
6149           printf (" %x: %s r%d,cpr0[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC0
6150         else printf (" %x: %s\n",start+i*4,insn[i]);
6151         break;
6152       case COP1:
6153         if(opcode2[i]<3)
6154           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC1
6155         else if(opcode2[i]>3)
6156           printf (" %x: %s r%d,cpr1[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC1
6157         else printf (" %x: %s\n",start+i*4,insn[i]);
6158         break;
6159       case COP2:
6160         if(opcode2[i]<3)
6161           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC2
6162         else if(opcode2[i]>3)
6163           printf (" %x: %s r%d,cpr2[%d]\n",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC2
6164         else printf (" %x: %s\n",start+i*4,insn[i]);
6165         break;
6166       case C1LS:
6167         printf (" %x: %s cpr1[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6168         break;
6169       case C2LS:
6170         printf (" %x: %s cpr2[%d],r%d+%x\n",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
6171         break;
6172       case INTCALL:
6173         printf (" %x: %s (INTCALL)\n",start+i*4,insn[i]);
6174         break;
6175       default:
6176         //printf (" %s %8x\n",insn[i],source[i]);
6177         printf (" %x: %s\n",start+i*4,insn[i]);
6178     }
6179 }
6180 #else
6181 static void disassemble_inst(int i) {}
6182 #endif // DISASM
6183
6184 #define DRC_TEST_VAL 0x74657374
6185
6186 static int new_dynarec_test(void)
6187 {
6188   int (*testfunc)(void) = (void *)out;
6189   void *beginning;
6190   int ret;
6191
6192   beginning = start_block();
6193   emit_movimm(DRC_TEST_VAL,0); // test
6194   emit_jmpreg(14);
6195   literal_pool(0);
6196   end_block(beginning);
6197   SysPrintf("testing if we can run recompiled code..\n");
6198   ret = testfunc();
6199   if (ret == DRC_TEST_VAL)
6200     SysPrintf("test passed.\n");
6201   else
6202     SysPrintf("test failed: %08x\n", ret);
6203   out = translation_cache;
6204   return ret == DRC_TEST_VAL;
6205 }
6206
6207 // clear the state completely, instead of just marking
6208 // things invalid like invalidate_all_pages() does
6209 void new_dynarec_clear_full()
6210 {
6211   int n;
6212   out = translation_cache;
6213   memset(invalid_code,1,sizeof(invalid_code));
6214   memset(hash_table,0xff,sizeof(hash_table));
6215   memset(mini_ht,-1,sizeof(mini_ht));
6216   memset(restore_candidate,0,sizeof(restore_candidate));
6217   memset(shadow,0,sizeof(shadow));
6218   copy=shadow;
6219   expirep=16384; // Expiry pointer, +2 blocks
6220   pending_exception=0;
6221   literalcount=0;
6222   stop_after_jal=0;
6223   inv_code_start=inv_code_end=~0;
6224   // TLB
6225   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6226   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6227   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6228 }
6229
6230 void new_dynarec_init()
6231 {
6232   SysPrintf("Init new dynarec\n");
6233
6234   // allocate/prepare a buffer for translation cache
6235   // see assem_arm.h for some explanation
6236 #if   defined(BASE_ADDR_FIXED)
6237   if (mmap(translation_cache, 1 << TARGET_SIZE_2,
6238             PROT_READ | PROT_WRITE | PROT_EXEC,
6239             MAP_PRIVATE | MAP_ANONYMOUS,
6240             -1, 0) != translation_cache) {
6241     SysPrintf("mmap() failed: %s\n", strerror(errno));
6242     SysPrintf("disable BASE_ADDR_FIXED and recompile\n");
6243     abort();
6244   }
6245 #elif defined(BASE_ADDR_DYNAMIC)
6246   #ifdef VITA
6247   sceBlock = sceKernelAllocMemBlockForVM("code", 1 << TARGET_SIZE_2);
6248   if (sceBlock < 0)
6249     SysPrintf("sceKernelAllocMemBlockForVM failed\n");
6250   int ret = sceKernelGetMemBlockBase(sceBlock, (void **)&translation_cache);
6251   if (ret < 0)
6252     SysPrintf("sceKernelGetMemBlockBase failed\n");
6253   #else
6254   translation_cache = mmap (NULL, 1 << TARGET_SIZE_2,
6255             PROT_READ | PROT_WRITE | PROT_EXEC,
6256             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
6257   if (translation_cache == MAP_FAILED) {
6258     SysPrintf("mmap() failed: %s\n", strerror(errno));
6259     abort();
6260   }
6261   #endif
6262 #else
6263   #ifndef NO_WRITE_EXEC
6264   // not all systems allow execute in data segment by default
6265   if (mprotect(translation_cache, 1<<TARGET_SIZE_2, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
6266     SysPrintf("mprotect() failed: %s\n", strerror(errno));
6267   #endif
6268 #endif
6269   out = translation_cache;
6270   cycle_multiplier=200;
6271   new_dynarec_clear_full();
6272 #ifdef HOST_IMM8
6273   // Copy this into local area so we don't have to put it in every literal pool
6274   invc_ptr=invalid_code;
6275 #endif
6276   arch_init();
6277   new_dynarec_test();
6278 #ifndef RAM_FIXED
6279   ram_offset=(uintptr_t)rdram-0x80000000;
6280 #endif
6281   if (ram_offset!=0)
6282     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
6283 }
6284
6285 void new_dynarec_cleanup()
6286 {
6287   int n;
6288 #if defined(BASE_ADDR_FIXED) || defined(BASE_ADDR_DYNAMIC)
6289   #ifdef VITA
6290   sceKernelFreeMemBlock(sceBlock);
6291   sceBlock = -1;
6292   #else
6293   if (munmap(translation_cache, 1<<TARGET_SIZE_2) < 0)
6294     SysPrintf("munmap() failed\n");
6295   #endif
6296 #endif
6297   for(n=0;n<4096;n++) ll_clear(jump_in+n);
6298   for(n=0;n<4096;n++) ll_clear(jump_out+n);
6299   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
6300   #ifdef ROM_COPY
6301   if (munmap (ROM_COPY, 67108864) < 0) {SysPrintf("munmap() failed\n");}
6302   #endif
6303 }
6304
6305 static u_int *get_source_start(u_int addr, u_int *limit)
6306 {
6307   if (addr < 0x00200000 ||
6308     (0xa0000000 <= addr && addr < 0xa0200000)) {
6309     // used for BIOS calls mostly?
6310     *limit = (addr&0xa0000000)|0x00200000;
6311     return (u_int *)(rdram + (addr&0x1fffff));
6312   }
6313   else if (!Config.HLE && (
6314     /* (0x9fc00000 <= addr && addr < 0x9fc80000) ||*/
6315     (0xbfc00000 <= addr && addr < 0xbfc80000))) {
6316     // BIOS
6317     *limit = (addr & 0xfff00000) | 0x80000;
6318     return (u_int *)((u_char *)psxR + (addr&0x7ffff));
6319   }
6320   else if (addr >= 0x80000000 && addr < 0x80000000+RAM_SIZE) {
6321     *limit = (addr & 0x80600000) + 0x00200000;
6322     return (u_int *)(rdram + (addr&0x1fffff));
6323   }
6324   return NULL;
6325 }
6326
6327 static u_int scan_for_ret(u_int addr)
6328 {
6329   u_int limit = 0;
6330   u_int *mem;
6331
6332   mem = get_source_start(addr, &limit);
6333   if (mem == NULL)
6334     return addr;
6335
6336   if (limit > addr + 0x1000)
6337     limit = addr + 0x1000;
6338   for (; addr < limit; addr += 4, mem++) {
6339     if (*mem == 0x03e00008) // jr $ra
6340       return addr + 8;
6341   }
6342   return addr;
6343 }
6344
6345 struct savestate_block {
6346   uint32_t addr;
6347   uint32_t regflags;
6348 };
6349
6350 static int addr_cmp(const void *p1_, const void *p2_)
6351 {
6352   const struct savestate_block *p1 = p1_, *p2 = p2_;
6353   return p1->addr - p2->addr;
6354 }
6355
6356 int new_dynarec_save_blocks(void *save, int size)
6357 {
6358   struct savestate_block *blocks = save;
6359   int maxcount = size / sizeof(blocks[0]);
6360   struct savestate_block tmp_blocks[1024];
6361   struct ll_entry *head;
6362   int p, s, d, o, bcnt;
6363   u_int addr;
6364
6365   o = 0;
6366   for (p = 0; p < ARRAY_SIZE(jump_in); p++) {
6367     bcnt = 0;
6368     for (head = jump_in[p]; head != NULL; head = head->next) {
6369       tmp_blocks[bcnt].addr = head->vaddr;
6370       tmp_blocks[bcnt].regflags = head->reg_sv_flags;
6371       bcnt++;
6372     }
6373     if (bcnt < 1)
6374       continue;
6375     qsort(tmp_blocks, bcnt, sizeof(tmp_blocks[0]), addr_cmp);
6376
6377     addr = tmp_blocks[0].addr;
6378     for (s = d = 0; s < bcnt; s++) {
6379       if (tmp_blocks[s].addr < addr)
6380         continue;
6381       if (d == 0 || tmp_blocks[d-1].addr != tmp_blocks[s].addr)
6382         tmp_blocks[d++] = tmp_blocks[s];
6383       addr = scan_for_ret(tmp_blocks[s].addr);
6384     }
6385
6386     if (o + d > maxcount)
6387       d = maxcount - o;
6388     memcpy(&blocks[o], tmp_blocks, d * sizeof(blocks[0]));
6389     o += d;
6390   }
6391
6392   return o * sizeof(blocks[0]);
6393 }
6394
6395 void new_dynarec_load_blocks(const void *save, int size)
6396 {
6397   const struct savestate_block *blocks = save;
6398   int count = size / sizeof(blocks[0]);
6399   u_int regs_save[32];
6400   uint32_t f;
6401   int i, b;
6402
6403   get_addr(psxRegs.pc);
6404
6405   // change GPRs for speculation to at least partially work..
6406   memcpy(regs_save, &psxRegs.GPR, sizeof(regs_save));
6407   for (i = 1; i < 32; i++)
6408     psxRegs.GPR.r[i] = 0x80000000;
6409
6410   for (b = 0; b < count; b++) {
6411     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6412       if (f & 1)
6413         psxRegs.GPR.r[i] = 0x1f800000;
6414     }
6415
6416     get_addr(blocks[b].addr);
6417
6418     for (f = blocks[b].regflags, i = 0; f; f >>= 1, i++) {
6419       if (f & 1)
6420         psxRegs.GPR.r[i] = 0x80000000;
6421     }
6422   }
6423
6424   memcpy(&psxRegs.GPR, regs_save, sizeof(regs_save));
6425 }
6426
6427 int new_recompile_block(int addr)
6428 {
6429   u_int pagelimit = 0;
6430   u_int state_rflags = 0;
6431   int i;
6432
6433   assem_debug("NOTCOMPILED: addr = %x -> %p\n", addr, out);
6434   //printf("TRACE: count=%d next=%d (compile %x)\n",Count,next_interupt,addr);
6435   //if(debug)
6436   //printf("fpu mapping=%x enabled=%x\n",(Status & 0x04000000)>>26,(Status & 0x20000000)>>29);
6437
6438   // this is just for speculation
6439   for (i = 1; i < 32; i++) {
6440     if ((psxRegs.GPR.r[i] & 0xffff0000) == 0x1f800000)
6441       state_rflags |= 1 << i;
6442   }
6443
6444   start = (u_int)addr&~3;
6445   //assert(((u_int)addr&1)==0);
6446   new_dynarec_did_compile=1;
6447   if (Config.HLE && start == 0x80001000) // hlecall
6448   {
6449     // XXX: is this enough? Maybe check hleSoftCall?
6450     void *beginning=start_block();
6451     u_int page=get_page(start);
6452
6453     invalid_code[start>>12]=0;
6454     emit_movimm(start,0);
6455     emit_writeword(0,&pcaddr);
6456     emit_jmp(new_dyna_leave);
6457     literal_pool(0);
6458     end_block(beginning);
6459     ll_add_flags(jump_in+page,start,state_rflags,(void *)beginning);
6460     return 0;
6461   }
6462
6463   source = get_source_start(start, &pagelimit);
6464   if (source == NULL) {
6465     SysPrintf("Compile at bogus memory address: %08x\n", addr);
6466     exit(1);
6467   }
6468
6469   /* Pass 1: disassemble */
6470   /* Pass 2: register dependencies, branch targets */
6471   /* Pass 3: register allocation */
6472   /* Pass 4: branch dependencies */
6473   /* Pass 5: pre-alloc */
6474   /* Pass 6: optimize clean/dirty state */
6475   /* Pass 7: flag 32-bit registers */
6476   /* Pass 8: assembly */
6477   /* Pass 9: linker */
6478   /* Pass 10: garbage collection / free memory */
6479
6480   int j;
6481   int done=0;
6482   unsigned int type,op,op2;
6483
6484   //printf("addr = %x source = %x %x\n", addr,source,source[0]);
6485
6486   /* Pass 1 disassembly */
6487
6488   for(i=0;!done;i++) {
6489     bt[i]=0;likely[i]=0;ooo[i]=0;op2=0;
6490     minimum_free_regs[i]=0;
6491     opcode[i]=op=source[i]>>26;
6492     switch(op)
6493     {
6494       case 0x00: strcpy(insn[i],"special"); type=NI;
6495         op2=source[i]&0x3f;
6496         switch(op2)
6497         {
6498           case 0x00: strcpy(insn[i],"SLL"); type=SHIFTIMM; break;
6499           case 0x02: strcpy(insn[i],"SRL"); type=SHIFTIMM; break;
6500           case 0x03: strcpy(insn[i],"SRA"); type=SHIFTIMM; break;
6501           case 0x04: strcpy(insn[i],"SLLV"); type=SHIFT; break;
6502           case 0x06: strcpy(insn[i],"SRLV"); type=SHIFT; break;
6503           case 0x07: strcpy(insn[i],"SRAV"); type=SHIFT; break;
6504           case 0x08: strcpy(insn[i],"JR"); type=RJUMP; break;
6505           case 0x09: strcpy(insn[i],"JALR"); type=RJUMP; break;
6506           case 0x0C: strcpy(insn[i],"SYSCALL"); type=SYSCALL; break;
6507           case 0x0D: strcpy(insn[i],"BREAK"); type=OTHER; break;
6508           case 0x0F: strcpy(insn[i],"SYNC"); type=OTHER; break;
6509           case 0x10: strcpy(insn[i],"MFHI"); type=MOV; break;
6510           case 0x11: strcpy(insn[i],"MTHI"); type=MOV; break;
6511           case 0x12: strcpy(insn[i],"MFLO"); type=MOV; break;
6512           case 0x13: strcpy(insn[i],"MTLO"); type=MOV; break;
6513           case 0x18: strcpy(insn[i],"MULT"); type=MULTDIV; break;
6514           case 0x19: strcpy(insn[i],"MULTU"); type=MULTDIV; break;
6515           case 0x1A: strcpy(insn[i],"DIV"); type=MULTDIV; break;
6516           case 0x1B: strcpy(insn[i],"DIVU"); type=MULTDIV; break;
6517           case 0x20: strcpy(insn[i],"ADD"); type=ALU; break;
6518           case 0x21: strcpy(insn[i],"ADDU"); type=ALU; break;
6519           case 0x22: strcpy(insn[i],"SUB"); type=ALU; break;
6520           case 0x23: strcpy(insn[i],"SUBU"); type=ALU; break;
6521           case 0x24: strcpy(insn[i],"AND"); type=ALU; break;
6522           case 0x25: strcpy(insn[i],"OR"); type=ALU; break;
6523           case 0x26: strcpy(insn[i],"XOR"); type=ALU; break;
6524           case 0x27: strcpy(insn[i],"NOR"); type=ALU; break;
6525           case 0x2A: strcpy(insn[i],"SLT"); type=ALU; break;
6526           case 0x2B: strcpy(insn[i],"SLTU"); type=ALU; break;
6527           case 0x30: strcpy(insn[i],"TGE"); type=NI; break;
6528           case 0x31: strcpy(insn[i],"TGEU"); type=NI; break;
6529           case 0x32: strcpy(insn[i],"TLT"); type=NI; break;
6530           case 0x33: strcpy(insn[i],"TLTU"); type=NI; break;
6531           case 0x34: strcpy(insn[i],"TEQ"); type=NI; break;
6532           case 0x36: strcpy(insn[i],"TNE"); type=NI; break;
6533 #if 0
6534           case 0x14: strcpy(insn[i],"DSLLV"); type=SHIFT; break;
6535           case 0x16: strcpy(insn[i],"DSRLV"); type=SHIFT; break;
6536           case 0x17: strcpy(insn[i],"DSRAV"); type=SHIFT; break;
6537           case 0x1C: strcpy(insn[i],"DMULT"); type=MULTDIV; break;
6538           case 0x1D: strcpy(insn[i],"DMULTU"); type=MULTDIV; break;
6539           case 0x1E: strcpy(insn[i],"DDIV"); type=MULTDIV; break;
6540           case 0x1F: strcpy(insn[i],"DDIVU"); type=MULTDIV; break;
6541           case 0x2C: strcpy(insn[i],"DADD"); type=ALU; break;
6542           case 0x2D: strcpy(insn[i],"DADDU"); type=ALU; break;
6543           case 0x2E: strcpy(insn[i],"DSUB"); type=ALU; break;
6544           case 0x2F: strcpy(insn[i],"DSUBU"); type=ALU; break;
6545           case 0x38: strcpy(insn[i],"DSLL"); type=SHIFTIMM; break;
6546           case 0x3A: strcpy(insn[i],"DSRL"); type=SHIFTIMM; break;
6547           case 0x3B: strcpy(insn[i],"DSRA"); type=SHIFTIMM; break;
6548           case 0x3C: strcpy(insn[i],"DSLL32"); type=SHIFTIMM; break;
6549           case 0x3E: strcpy(insn[i],"DSRL32"); type=SHIFTIMM; break;
6550           case 0x3F: strcpy(insn[i],"DSRA32"); type=SHIFTIMM; break;
6551 #endif
6552         }
6553         break;
6554       case 0x01: strcpy(insn[i],"regimm"); type=NI;
6555         op2=(source[i]>>16)&0x1f;
6556         switch(op2)
6557         {
6558           case 0x00: strcpy(insn[i],"BLTZ"); type=SJUMP; break;
6559           case 0x01: strcpy(insn[i],"BGEZ"); type=SJUMP; break;
6560           case 0x02: strcpy(insn[i],"BLTZL"); type=SJUMP; break;
6561           case 0x03: strcpy(insn[i],"BGEZL"); type=SJUMP; break;
6562           case 0x08: strcpy(insn[i],"TGEI"); type=NI; break;
6563           case 0x09: strcpy(insn[i],"TGEIU"); type=NI; break;
6564           case 0x0A: strcpy(insn[i],"TLTI"); type=NI; break;
6565           case 0x0B: strcpy(insn[i],"TLTIU"); type=NI; break;
6566           case 0x0C: strcpy(insn[i],"TEQI"); type=NI; break;
6567           case 0x0E: strcpy(insn[i],"TNEI"); type=NI; break;
6568           case 0x10: strcpy(insn[i],"BLTZAL"); type=SJUMP; break;
6569           case 0x11: strcpy(insn[i],"BGEZAL"); type=SJUMP; break;
6570           case 0x12: strcpy(insn[i],"BLTZALL"); type=SJUMP; break;
6571           case 0x13: strcpy(insn[i],"BGEZALL"); type=SJUMP; break;
6572         }
6573         break;
6574       case 0x02: strcpy(insn[i],"J"); type=UJUMP; break;
6575       case 0x03: strcpy(insn[i],"JAL"); type=UJUMP; break;
6576       case 0x04: strcpy(insn[i],"BEQ"); type=CJUMP; break;
6577       case 0x05: strcpy(insn[i],"BNE"); type=CJUMP; break;
6578       case 0x06: strcpy(insn[i],"BLEZ"); type=CJUMP; break;
6579       case 0x07: strcpy(insn[i],"BGTZ"); type=CJUMP; break;
6580       case 0x08: strcpy(insn[i],"ADDI"); type=IMM16; break;
6581       case 0x09: strcpy(insn[i],"ADDIU"); type=IMM16; break;
6582       case 0x0A: strcpy(insn[i],"SLTI"); type=IMM16; break;
6583       case 0x0B: strcpy(insn[i],"SLTIU"); type=IMM16; break;
6584       case 0x0C: strcpy(insn[i],"ANDI"); type=IMM16; break;
6585       case 0x0D: strcpy(insn[i],"ORI"); type=IMM16; break;
6586       case 0x0E: strcpy(insn[i],"XORI"); type=IMM16; break;
6587       case 0x0F: strcpy(insn[i],"LUI"); type=IMM16; break;
6588       case 0x10: strcpy(insn[i],"cop0"); type=NI;
6589         op2=(source[i]>>21)&0x1f;
6590         switch(op2)
6591         {
6592           case 0x00: strcpy(insn[i],"MFC0"); type=COP0; break;
6593           case 0x02: strcpy(insn[i],"CFC0"); type=COP0; break;
6594           case 0x04: strcpy(insn[i],"MTC0"); type=COP0; break;
6595           case 0x06: strcpy(insn[i],"CTC0"); type=COP0; break;
6596           case 0x10: strcpy(insn[i],"RFE"); type=COP0; break;
6597         }
6598         break;
6599       case 0x11: strcpy(insn[i],"cop1"); type=COP1;
6600         op2=(source[i]>>21)&0x1f;
6601         break;
6602 #if 0
6603       case 0x14: strcpy(insn[i],"BEQL"); type=CJUMP; break;
6604       case 0x15: strcpy(insn[i],"BNEL"); type=CJUMP; break;
6605       case 0x16: strcpy(insn[i],"BLEZL"); type=CJUMP; break;
6606       case 0x17: strcpy(insn[i],"BGTZL"); type=CJUMP; break;
6607       case 0x18: strcpy(insn[i],"DADDI"); type=IMM16; break;
6608       case 0x19: strcpy(insn[i],"DADDIU"); type=IMM16; break;
6609       case 0x1A: strcpy(insn[i],"LDL"); type=LOADLR; break;
6610       case 0x1B: strcpy(insn[i],"LDR"); type=LOADLR; break;
6611 #endif
6612       case 0x20: strcpy(insn[i],"LB"); type=LOAD; break;
6613       case 0x21: strcpy(insn[i],"LH"); type=LOAD; break;
6614       case 0x22: strcpy(insn[i],"LWL"); type=LOADLR; break;
6615       case 0x23: strcpy(insn[i],"LW"); type=LOAD; break;
6616       case 0x24: strcpy(insn[i],"LBU"); type=LOAD; break;
6617       case 0x25: strcpy(insn[i],"LHU"); type=LOAD; break;
6618       case 0x26: strcpy(insn[i],"LWR"); type=LOADLR; break;
6619 #if 0
6620       case 0x27: strcpy(insn[i],"LWU"); type=LOAD; break;
6621 #endif
6622       case 0x28: strcpy(insn[i],"SB"); type=STORE; break;
6623       case 0x29: strcpy(insn[i],"SH"); type=STORE; break;
6624       case 0x2A: strcpy(insn[i],"SWL"); type=STORELR; break;
6625       case 0x2B: strcpy(insn[i],"SW"); type=STORE; break;
6626 #if 0
6627       case 0x2C: strcpy(insn[i],"SDL"); type=STORELR; break;
6628       case 0x2D: strcpy(insn[i],"SDR"); type=STORELR; break;
6629 #endif
6630       case 0x2E: strcpy(insn[i],"SWR"); type=STORELR; break;
6631       case 0x2F: strcpy(insn[i],"CACHE"); type=NOP; break;
6632       case 0x30: strcpy(insn[i],"LL"); type=NI; break;
6633       case 0x31: strcpy(insn[i],"LWC1"); type=C1LS; break;
6634 #if 0
6635       case 0x34: strcpy(insn[i],"LLD"); type=NI; break;
6636       case 0x35: strcpy(insn[i],"LDC1"); type=C1LS; break;
6637       case 0x37: strcpy(insn[i],"LD"); type=LOAD; break;
6638 #endif
6639       case 0x38: strcpy(insn[i],"SC"); type=NI; break;
6640       case 0x39: strcpy(insn[i],"SWC1"); type=C1LS; break;
6641 #if 0
6642       case 0x3C: strcpy(insn[i],"SCD"); type=NI; break;
6643       case 0x3D: strcpy(insn[i],"SDC1"); type=C1LS; break;
6644       case 0x3F: strcpy(insn[i],"SD"); type=STORE; break;
6645 #endif
6646       case 0x12: strcpy(insn[i],"COP2"); type=NI;
6647         op2=(source[i]>>21)&0x1f;
6648         //if (op2 & 0x10) {
6649         if (source[i]&0x3f) { // use this hack to support old savestates with patched gte insns
6650           if (gte_handlers[source[i]&0x3f]!=NULL) {
6651             if (gte_regnames[source[i]&0x3f]!=NULL)
6652               strcpy(insn[i],gte_regnames[source[i]&0x3f]);
6653             else
6654               snprintf(insn[i], sizeof(insn[i]), "COP2 %x", source[i]&0x3f);
6655             type=C2OP;
6656           }
6657         }
6658         else switch(op2)
6659         {
6660           case 0x00: strcpy(insn[i],"MFC2"); type=COP2; break;
6661           case 0x02: strcpy(insn[i],"CFC2"); type=COP2; break;
6662           case 0x04: strcpy(insn[i],"MTC2"); type=COP2; break;
6663           case 0x06: strcpy(insn[i],"CTC2"); type=COP2; break;
6664         }
6665         break;
6666       case 0x32: strcpy(insn[i],"LWC2"); type=C2LS; break;
6667       case 0x3A: strcpy(insn[i],"SWC2"); type=C2LS; break;
6668       case 0x3B: strcpy(insn[i],"HLECALL"); type=HLECALL; break;
6669       default: strcpy(insn[i],"???"); type=NI;
6670         SysPrintf("NI %08x @%08x (%08x)\n", source[i], addr + i*4, addr);
6671         break;
6672     }
6673     itype[i]=type;
6674     opcode2[i]=op2;
6675     /* Get registers/immediates */
6676     lt1[i]=0;
6677     us1[i]=0;
6678     us2[i]=0;
6679     dep1[i]=0;
6680     dep2[i]=0;
6681     gte_rs[i]=gte_rt[i]=0;
6682     switch(type) {
6683       case LOAD:
6684         rs1[i]=(source[i]>>21)&0x1f;
6685         rs2[i]=0;
6686         rt1[i]=(source[i]>>16)&0x1f;
6687         rt2[i]=0;
6688         imm[i]=(short)source[i];
6689         break;
6690       case STORE:
6691       case STORELR:
6692         rs1[i]=(source[i]>>21)&0x1f;
6693         rs2[i]=(source[i]>>16)&0x1f;
6694         rt1[i]=0;
6695         rt2[i]=0;
6696         imm[i]=(short)source[i];
6697         if(op==0x2c||op==0x2d||op==0x3f) us1[i]=rs2[i]; // 64-bit SDL/SDR/SD
6698         break;
6699       case LOADLR:
6700         // LWL/LWR only load part of the register,
6701         // therefore the target register must be treated as a source too
6702         rs1[i]=(source[i]>>21)&0x1f;
6703         rs2[i]=(source[i]>>16)&0x1f;
6704         rt1[i]=(source[i]>>16)&0x1f;
6705         rt2[i]=0;
6706         imm[i]=(short)source[i];
6707         if(op==0x1a||op==0x1b) us1[i]=rs2[i]; // LDR/LDL
6708         if(op==0x26) dep1[i]=rt1[i]; // LWR
6709         break;
6710       case IMM16:
6711         if (op==0x0f) rs1[i]=0; // LUI instruction has no source register
6712         else rs1[i]=(source[i]>>21)&0x1f;
6713         rs2[i]=0;
6714         rt1[i]=(source[i]>>16)&0x1f;
6715         rt2[i]=0;
6716         if(op>=0x0c&&op<=0x0e) { // ANDI/ORI/XORI
6717           imm[i]=(unsigned short)source[i];
6718         }else{
6719           imm[i]=(short)source[i];
6720         }
6721         if(op==0x18||op==0x19) us1[i]=rs1[i]; // DADDI/DADDIU
6722         if(op==0x0a||op==0x0b) us1[i]=rs1[i]; // SLTI/SLTIU
6723         if(op==0x0d||op==0x0e) dep1[i]=rs1[i]; // ORI/XORI
6724         break;
6725       case UJUMP:
6726         rs1[i]=0;
6727         rs2[i]=0;
6728         rt1[i]=0;
6729         rt2[i]=0;
6730         // The JAL instruction writes to r31.
6731         if (op&1) {
6732           rt1[i]=31;
6733         }
6734         rs2[i]=CCREG;
6735         break;
6736       case RJUMP:
6737         rs1[i]=(source[i]>>21)&0x1f;
6738         rs2[i]=0;
6739         rt1[i]=0;
6740         rt2[i]=0;
6741         // The JALR instruction writes to rd.
6742         if (op2&1) {
6743           rt1[i]=(source[i]>>11)&0x1f;
6744         }
6745         rs2[i]=CCREG;
6746         break;
6747       case CJUMP:
6748         rs1[i]=(source[i]>>21)&0x1f;
6749         rs2[i]=(source[i]>>16)&0x1f;
6750         rt1[i]=0;
6751         rt2[i]=0;
6752         if(op&2) { // BGTZ/BLEZ
6753           rs2[i]=0;
6754         }
6755         us1[i]=rs1[i];
6756         us2[i]=rs2[i];
6757         likely[i]=op>>4;
6758         break;
6759       case SJUMP:
6760         rs1[i]=(source[i]>>21)&0x1f;
6761         rs2[i]=CCREG;
6762         rt1[i]=0;
6763         rt2[i]=0;
6764         us1[i]=rs1[i];
6765         if(op2&0x10) { // BxxAL
6766           rt1[i]=31;
6767           // NOTE: If the branch is not taken, r31 is still overwritten
6768         }
6769         likely[i]=(op2&2)>>1;
6770         break;
6771       case ALU:
6772         rs1[i]=(source[i]>>21)&0x1f; // source
6773         rs2[i]=(source[i]>>16)&0x1f; // subtract amount
6774         rt1[i]=(source[i]>>11)&0x1f; // destination
6775         rt2[i]=0;
6776         if(op2==0x2a||op2==0x2b) { // SLT/SLTU
6777           us1[i]=rs1[i];us2[i]=rs2[i];
6778         }
6779         else if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
6780           dep1[i]=rs1[i];dep2[i]=rs2[i];
6781         }
6782         else if(op2>=0x2c&&op2<=0x2f) { // DADD/DSUB
6783           dep1[i]=rs1[i];dep2[i]=rs2[i];
6784         }
6785         break;
6786       case MULTDIV:
6787         rs1[i]=(source[i]>>21)&0x1f; // source
6788         rs2[i]=(source[i]>>16)&0x1f; // divisor
6789         rt1[i]=HIREG;
6790         rt2[i]=LOREG;
6791         if (op2>=0x1c&&op2<=0x1f) { // DMULT/DMULTU/DDIV/DDIVU
6792           us1[i]=rs1[i];us2[i]=rs2[i];
6793         }
6794         break;
6795       case MOV:
6796         rs1[i]=0;
6797         rs2[i]=0;
6798         rt1[i]=0;
6799         rt2[i]=0;
6800         if(op2==0x10) rs1[i]=HIREG; // MFHI
6801         if(op2==0x11) rt1[i]=HIREG; // MTHI
6802         if(op2==0x12) rs1[i]=LOREG; // MFLO
6803         if(op2==0x13) rt1[i]=LOREG; // MTLO
6804         if((op2&0x1d)==0x10) rt1[i]=(source[i]>>11)&0x1f; // MFxx
6805         if((op2&0x1d)==0x11) rs1[i]=(source[i]>>21)&0x1f; // MTxx
6806         dep1[i]=rs1[i];
6807         break;
6808       case SHIFT:
6809         rs1[i]=(source[i]>>16)&0x1f; // target of shift
6810         rs2[i]=(source[i]>>21)&0x1f; // shift amount
6811         rt1[i]=(source[i]>>11)&0x1f; // destination
6812         rt2[i]=0;
6813         // DSLLV/DSRLV/DSRAV are 64-bit
6814         if(op2>=0x14&&op2<=0x17) us1[i]=rs1[i];
6815         break;
6816       case SHIFTIMM:
6817         rs1[i]=(source[i]>>16)&0x1f;
6818         rs2[i]=0;
6819         rt1[i]=(source[i]>>11)&0x1f;
6820         rt2[i]=0;
6821         imm[i]=(source[i]>>6)&0x1f;
6822         // DSxx32 instructions
6823         if(op2>=0x3c) imm[i]|=0x20;
6824         // DSLL/DSRL/DSRA/DSRA32/DSRL32 but not DSLL32 require 64-bit source
6825         if(op2>=0x38&&op2!=0x3c) us1[i]=rs1[i];
6826         break;
6827       case COP0:
6828         rs1[i]=0;
6829         rs2[i]=0;
6830         rt1[i]=0;
6831         rt2[i]=0;
6832         if(op2==0||op2==2) rt1[i]=(source[i]>>16)&0x1F; // MFC0/CFC0
6833         if(op2==4||op2==6) rs1[i]=(source[i]>>16)&0x1F; // MTC0/CTC0
6834         if(op2==4&&((source[i]>>11)&0x1f)==12) rt2[i]=CSREG; // Status
6835         if(op2==16) if((source[i]&0x3f)==0x18) rs2[i]=CCREG; // ERET
6836         break;
6837       case COP1:
6838         rs1[i]=0;
6839         rs2[i]=0;
6840         rt1[i]=0;
6841         rt2[i]=0;
6842         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC1/DMFC1/CFC1
6843         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC1/DMTC1/CTC1
6844         if(op2==5) us1[i]=rs1[i]; // DMTC1
6845         rs2[i]=CSREG;
6846         break;
6847       case COP2:
6848         rs1[i]=0;
6849         rs2[i]=0;
6850         rt1[i]=0;
6851         rt2[i]=0;
6852         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC2/CFC2
6853         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC2/CTC2
6854         rs2[i]=CSREG;
6855         int gr=(source[i]>>11)&0x1F;
6856         switch(op2)
6857         {
6858           case 0x00: gte_rs[i]=1ll<<gr; break; // MFC2
6859           case 0x04: gte_rt[i]=1ll<<gr; break; // MTC2
6860           case 0x02: gte_rs[i]=1ll<<(gr+32); break; // CFC2
6861           case 0x06: gte_rt[i]=1ll<<(gr+32); break; // CTC2
6862         }
6863         break;
6864       case C1LS:
6865         rs1[i]=(source[i]>>21)&0x1F;
6866         rs2[i]=CSREG;
6867         rt1[i]=0;
6868         rt2[i]=0;
6869         imm[i]=(short)source[i];
6870         break;
6871       case C2LS:
6872         rs1[i]=(source[i]>>21)&0x1F;
6873         rs2[i]=0;
6874         rt1[i]=0;
6875         rt2[i]=0;
6876         imm[i]=(short)source[i];
6877         if(op==0x32) gte_rt[i]=1ll<<((source[i]>>16)&0x1F); // LWC2
6878         else gte_rs[i]=1ll<<((source[i]>>16)&0x1F); // SWC2
6879         break;
6880       case C2OP:
6881         rs1[i]=0;
6882         rs2[i]=0;
6883         rt1[i]=0;
6884         rt2[i]=0;
6885         gte_rs[i]=gte_reg_reads[source[i]&0x3f];
6886         gte_rt[i]=gte_reg_writes[source[i]&0x3f];
6887         gte_rt[i]|=1ll<<63; // every op changes flags
6888         if((source[i]&0x3f)==GTE_MVMVA) {
6889           int v = (source[i] >> 15) & 3;
6890           gte_rs[i]&=~0xe3fll;
6891           if(v==3) gte_rs[i]|=0xe00ll;
6892           else gte_rs[i]|=3ll<<(v*2);
6893         }
6894         break;
6895       case SYSCALL:
6896       case HLECALL:
6897       case INTCALL:
6898         rs1[i]=CCREG;
6899         rs2[i]=0;
6900         rt1[i]=0;
6901         rt2[i]=0;
6902         break;
6903       default:
6904         rs1[i]=0;
6905         rs2[i]=0;
6906         rt1[i]=0;
6907         rt2[i]=0;
6908     }
6909     /* Calculate branch target addresses */
6910     if(type==UJUMP)
6911       ba[i]=((start+i*4+4)&0xF0000000)|(((unsigned int)source[i]<<6)>>4);
6912     else if(type==CJUMP&&rs1[i]==rs2[i]&&(op&1))
6913       ba[i]=start+i*4+8; // Ignore never taken branch
6914     else if(type==SJUMP&&rs1[i]==0&&!(op2&1))
6915       ba[i]=start+i*4+8; // Ignore never taken branch
6916     else if(type==CJUMP||type==SJUMP)
6917       ba[i]=start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14);
6918     else ba[i]=-1;
6919     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP)) {
6920       int do_in_intrp=0;
6921       // branch in delay slot?
6922       if(type==RJUMP||type==UJUMP||type==CJUMP||type==SJUMP) {
6923         // don't handle first branch and call interpreter if it's hit
6924         SysPrintf("branch in delay slot @%08x (%08x)\n", addr + i*4, addr);
6925         do_in_intrp=1;
6926       }
6927       // basic load delay detection
6928       else if((type==LOAD||type==LOADLR||type==COP0||type==COP2||type==C2LS)&&rt1[i]!=0) {
6929         int t=(ba[i-1]-start)/4;
6930         if(0 <= t && t < i &&(rt1[i]==rs1[t]||rt1[i]==rs2[t])&&itype[t]!=CJUMP&&itype[t]!=SJUMP) {
6931           // jump target wants DS result - potential load delay effect
6932           SysPrintf("load delay @%08x (%08x)\n", addr + i*4, addr);
6933           do_in_intrp=1;
6934           bt[t+1]=1; // expected return from interpreter
6935         }
6936         else if(i>=2&&rt1[i-2]==2&&rt1[i]==2&&rs1[i]!=2&&rs2[i]!=2&&rs1[i-1]!=2&&rs2[i-1]!=2&&
6937               !(i>=3&&(itype[i-3]==RJUMP||itype[i-3]==UJUMP||itype[i-3]==CJUMP||itype[i-3]==SJUMP))) {
6938           // v0 overwrite like this is a sign of trouble, bail out
6939           SysPrintf("v0 overwrite @%08x (%08x)\n", addr + i*4, addr);
6940           do_in_intrp=1;
6941         }
6942       }
6943       if(do_in_intrp) {
6944         rs1[i-1]=CCREG;
6945         rs2[i-1]=rt1[i-1]=rt2[i-1]=0;
6946         ba[i-1]=-1;
6947         itype[i-1]=INTCALL;
6948         done=2;
6949         i--; // don't compile the DS
6950       }
6951     }
6952     /* Is this the end of the block? */
6953     if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)) {
6954       if(rt1[i-1]==0) { // Continue past subroutine call (JAL)
6955         done=2;
6956       }
6957       else {
6958         if(stop_after_jal) done=1;
6959         // Stop on BREAK
6960         if((source[i+1]&0xfc00003f)==0x0d) done=1;
6961       }
6962       // Don't recompile stuff that's already compiled
6963       if(check_addr(start+i*4+4)) done=1;
6964       // Don't get too close to the limit
6965       if(i>MAXBLOCK/2) done=1;
6966     }
6967     if(itype[i]==SYSCALL&&stop_after_jal) done=1;
6968     if(itype[i]==HLECALL||itype[i]==INTCALL) done=2;
6969     if(done==2) {
6970       // Does the block continue due to a branch?
6971       for(j=i-1;j>=0;j--)
6972       {
6973         if(ba[j]==start+i*4) done=j=0; // Branch into delay slot
6974         if(ba[j]==start+i*4+4) done=j=0;
6975         if(ba[j]==start+i*4+8) done=j=0;
6976       }
6977     }
6978     //assert(i<MAXBLOCK-1);
6979     if(start+i*4==pagelimit-4) done=1;
6980     assert(start+i*4<pagelimit);
6981     if (i==MAXBLOCK-1) done=1;
6982     // Stop if we're compiling junk
6983     if(itype[i]==NI&&opcode[i]==0x11) {
6984       done=stop_after_jal=1;
6985       SysPrintf("Disabled speculative precompilation\n");
6986     }
6987   }
6988   slen=i;
6989   if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==RJUMP) {
6990     if(start+i*4==pagelimit) {
6991       itype[i-1]=SPAN;
6992     }
6993   }
6994   assert(slen>0);
6995
6996   /* Pass 2 - Register dependencies and branch targets */
6997
6998   unneeded_registers(0,slen-1,0);
6999
7000   /* Pass 3 - Register allocation */
7001
7002   struct regstat current; // Current register allocations/status
7003   current.dirty=0;
7004   current.u=unneeded_reg[0];
7005   clear_all_regs(current.regmap);
7006   alloc_reg(&current,0,CCREG);
7007   dirty_reg(&current,CCREG);
7008   current.isconst=0;
7009   current.wasconst=0;
7010   current.waswritten=0;
7011   int ds=0;
7012   int cc=0;
7013   int hr=-1;
7014
7015   if((u_int)addr&1) {
7016     // First instruction is delay slot
7017     cc=-1;
7018     bt[1]=1;
7019     ds=1;
7020     unneeded_reg[0]=1;
7021     current.regmap[HOST_BTREG]=BTREG;
7022   }
7023
7024   for(i=0;i<slen;i++)
7025   {
7026     if(bt[i])
7027     {
7028       int hr;
7029       for(hr=0;hr<HOST_REGS;hr++)
7030       {
7031         // Is this really necessary?
7032         if(current.regmap[hr]==0) current.regmap[hr]=-1;
7033       }
7034       current.isconst=0;
7035       current.waswritten=0;
7036     }
7037     if(i>1)
7038     {
7039       if((opcode[i-2]&0x2f)==0x05) // BNE/BNEL
7040       {
7041         if(rs1[i-2]==0||rs2[i-2]==0)
7042         {
7043           if(rs1[i-2]) {
7044             int hr=get_reg(current.regmap,rs1[i-2]|64);
7045             if(hr>=0) current.regmap[hr]=-1;
7046           }
7047           if(rs2[i-2]) {
7048             int hr=get_reg(current.regmap,rs2[i-2]|64);
7049             if(hr>=0) current.regmap[hr]=-1;
7050           }
7051         }
7052       }
7053     }
7054
7055     memcpy(regmap_pre[i],current.regmap,sizeof(current.regmap));
7056     regs[i].wasconst=current.isconst;
7057     regs[i].wasdirty=current.dirty;
7058     regs[i].loadedconst=0;
7059     if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP) {
7060       if(i+1<slen) {
7061         current.u=unneeded_reg[i+1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7062         current.u|=1;
7063       } else {
7064         current.u=1;
7065       }
7066     } else {
7067       if(i+1<slen) {
7068         current.u=branch_unneeded_reg[i]&~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
7069         current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7070         current.u|=1;
7071       } else { SysPrintf("oops, branch at end of block with no delay slot\n");exit(1); }
7072     }
7073     is_ds[i]=ds;
7074     if(ds) {
7075       ds=0; // Skip delay slot, already allocated as part of branch
7076       // ...but we need to alloc it in case something jumps here
7077       if(i+1<slen) {
7078         current.u=branch_unneeded_reg[i-1]&unneeded_reg[i+1];
7079       }else{
7080         current.u=branch_unneeded_reg[i-1];
7081       }
7082       current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
7083       current.u|=1;
7084       struct regstat temp;
7085       memcpy(&temp,&current,sizeof(current));
7086       temp.wasdirty=temp.dirty;
7087       // TODO: Take into account unconditional branches, as below
7088       delayslot_alloc(&temp,i);
7089       memcpy(regs[i].regmap,temp.regmap,sizeof(temp.regmap));
7090       regs[i].wasdirty=temp.wasdirty;
7091       regs[i].dirty=temp.dirty;
7092       regs[i].isconst=0;
7093       regs[i].wasconst=0;
7094       current.isconst=0;
7095       // Create entry (branch target) regmap
7096       for(hr=0;hr<HOST_REGS;hr++)
7097       {
7098         int r=temp.regmap[hr];
7099         if(r>=0) {
7100           if(r!=regmap_pre[i][hr]) {
7101             regs[i].regmap_entry[hr]=-1;
7102           }
7103           else
7104           {
7105             if(r<64){
7106               if((current.u>>r)&1) {
7107                 regs[i].regmap_entry[hr]=-1;
7108                 regs[i].regmap[hr]=-1;
7109                 //Don't clear regs in the delay slot as the branch might need them
7110                 //current.regmap[hr]=-1;
7111               }else
7112                 regs[i].regmap_entry[hr]=r;
7113             }
7114             else {
7115               assert(0);
7116             }
7117           }
7118         } else {
7119           // First instruction expects CCREG to be allocated
7120           if(i==0&&hr==HOST_CCREG)
7121             regs[i].regmap_entry[hr]=CCREG;
7122           else
7123             regs[i].regmap_entry[hr]=-1;
7124         }
7125       }
7126     }
7127     else { // Not delay slot
7128       switch(itype[i]) {
7129         case UJUMP:
7130           //current.isconst=0; // DEBUG
7131           //current.wasconst=0; // DEBUG
7132           //regs[i].wasconst=0; // DEBUG
7133           clear_const(&current,rt1[i]);
7134           alloc_cc(&current,i);
7135           dirty_reg(&current,CCREG);
7136           if (rt1[i]==31) {
7137             alloc_reg(&current,i,31);
7138             dirty_reg(&current,31);
7139             //assert(rs1[i+1]!=31&&rs2[i+1]!=31);
7140             //assert(rt1[i+1]!=rt1[i]);
7141             #ifdef REG_PREFETCH
7142             alloc_reg(&current,i,PTEMP);
7143             #endif
7144           }
7145           ooo[i]=1;
7146           delayslot_alloc(&current,i+1);
7147           //current.isconst=0; // DEBUG
7148           ds=1;
7149           //printf("i=%d, isconst=%x\n",i,current.isconst);
7150           break;
7151         case RJUMP:
7152           //current.isconst=0;
7153           //current.wasconst=0;
7154           //regs[i].wasconst=0;
7155           clear_const(&current,rs1[i]);
7156           clear_const(&current,rt1[i]);
7157           alloc_cc(&current,i);
7158           dirty_reg(&current,CCREG);
7159           if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
7160             alloc_reg(&current,i,rs1[i]);
7161             if (rt1[i]!=0) {
7162               alloc_reg(&current,i,rt1[i]);
7163               dirty_reg(&current,rt1[i]);
7164               assert(rs1[i+1]!=rt1[i]&&rs2[i+1]!=rt1[i]);
7165               assert(rt1[i+1]!=rt1[i]);
7166               #ifdef REG_PREFETCH
7167               alloc_reg(&current,i,PTEMP);
7168               #endif
7169             }
7170             #ifdef USE_MINI_HT
7171             if(rs1[i]==31) { // JALR
7172               alloc_reg(&current,i,RHASH);
7173               alloc_reg(&current,i,RHTBL);
7174             }
7175             #endif
7176             delayslot_alloc(&current,i+1);
7177           } else {
7178             // The delay slot overwrites our source register,
7179             // allocate a temporary register to hold the old value.
7180             current.isconst=0;
7181             current.wasconst=0;
7182             regs[i].wasconst=0;
7183             delayslot_alloc(&current,i+1);
7184             current.isconst=0;
7185             alloc_reg(&current,i,RTEMP);
7186           }
7187           //current.isconst=0; // DEBUG
7188           ooo[i]=1;
7189           ds=1;
7190           break;
7191         case CJUMP:
7192           //current.isconst=0;
7193           //current.wasconst=0;
7194           //regs[i].wasconst=0;
7195           clear_const(&current,rs1[i]);
7196           clear_const(&current,rs2[i]);
7197           if((opcode[i]&0x3E)==4) // BEQ/BNE
7198           {
7199             alloc_cc(&current,i);
7200             dirty_reg(&current,CCREG);
7201             if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7202             if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7203             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]))||
7204                (rs2[i]&&(rs2[i]==rt1[i+1]||rs2[i]==rt2[i+1]))) {
7205               // The delay slot overwrites one of our conditions.
7206               // Allocate the branch condition registers instead.
7207               current.isconst=0;
7208               current.wasconst=0;
7209               regs[i].wasconst=0;
7210               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7211               if(rs2[i]) alloc_reg(&current,i,rs2[i]);
7212             }
7213             else
7214             {
7215               ooo[i]=1;
7216               delayslot_alloc(&current,i+1);
7217             }
7218           }
7219           else
7220           if((opcode[i]&0x3E)==6) // BLEZ/BGTZ
7221           {
7222             alloc_cc(&current,i);
7223             dirty_reg(&current,CCREG);
7224             alloc_reg(&current,i,rs1[i]);
7225             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
7226               // The delay slot overwrites one of our conditions.
7227               // Allocate the branch condition registers instead.
7228               current.isconst=0;
7229               current.wasconst=0;
7230               regs[i].wasconst=0;
7231               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7232             }
7233             else
7234             {
7235               ooo[i]=1;
7236               delayslot_alloc(&current,i+1);
7237             }
7238           }
7239           else
7240           // Don't alloc the delay slot yet because we might not execute it
7241           if((opcode[i]&0x3E)==0x14) // BEQL/BNEL
7242           {
7243             current.isconst=0;
7244             current.wasconst=0;
7245             regs[i].wasconst=0;
7246             alloc_cc(&current,i);
7247             dirty_reg(&current,CCREG);
7248             alloc_reg(&current,i,rs1[i]);
7249             alloc_reg(&current,i,rs2[i]);
7250           }
7251           else
7252           if((opcode[i]&0x3E)==0x16) // BLEZL/BGTZL
7253           {
7254             current.isconst=0;
7255             current.wasconst=0;
7256             regs[i].wasconst=0;
7257             alloc_cc(&current,i);
7258             dirty_reg(&current,CCREG);
7259             alloc_reg(&current,i,rs1[i]);
7260           }
7261           ds=1;
7262           //current.isconst=0;
7263           break;
7264         case SJUMP:
7265           //current.isconst=0;
7266           //current.wasconst=0;
7267           //regs[i].wasconst=0;
7268           clear_const(&current,rs1[i]);
7269           clear_const(&current,rt1[i]);
7270           //if((opcode2[i]&0x1E)==0x0) // BLTZ/BGEZ
7271           if((opcode2[i]&0x0E)==0x0) // BLTZ/BGEZ
7272           {
7273             alloc_cc(&current,i);
7274             dirty_reg(&current,CCREG);
7275             alloc_reg(&current,i,rs1[i]);
7276             if (rt1[i]==31) { // BLTZAL/BGEZAL
7277               alloc_reg(&current,i,31);
7278               dirty_reg(&current,31);
7279               //#ifdef REG_PREFETCH
7280               //alloc_reg(&current,i,PTEMP);
7281               //#endif
7282             }
7283             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) // The delay slot overwrites the branch condition.
7284                ||(rt1[i]==31&&(rs1[i+1]==31||rs2[i+1]==31||rt1[i+1]==31||rt2[i+1]==31))) { // DS touches $ra
7285               // Allocate the branch condition registers instead.
7286               current.isconst=0;
7287               current.wasconst=0;
7288               regs[i].wasconst=0;
7289               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
7290             }
7291             else
7292             {
7293               ooo[i]=1;
7294               delayslot_alloc(&current,i+1);
7295             }
7296           }
7297           else
7298           // Don't alloc the delay slot yet because we might not execute it
7299           if((opcode2[i]&0x1E)==0x2) // BLTZL/BGEZL
7300           {
7301             current.isconst=0;
7302             current.wasconst=0;
7303             regs[i].wasconst=0;
7304             alloc_cc(&current,i);
7305             dirty_reg(&current,CCREG);
7306             alloc_reg(&current,i,rs1[i]);
7307           }
7308           ds=1;
7309           //current.isconst=0;
7310           break;
7311         case IMM16:
7312           imm16_alloc(&current,i);
7313           break;
7314         case LOAD:
7315         case LOADLR:
7316           load_alloc(&current,i);
7317           break;
7318         case STORE:
7319         case STORELR:
7320           store_alloc(&current,i);
7321           break;
7322         case ALU:
7323           alu_alloc(&current,i);
7324           break;
7325         case SHIFT:
7326           shift_alloc(&current,i);
7327           break;
7328         case MULTDIV:
7329           multdiv_alloc(&current,i);
7330           break;
7331         case SHIFTIMM:
7332           shiftimm_alloc(&current,i);
7333           break;
7334         case MOV:
7335           mov_alloc(&current,i);
7336           break;
7337         case COP0:
7338           cop0_alloc(&current,i);
7339           break;
7340         case COP1:
7341         case COP2:
7342           cop12_alloc(&current,i);
7343           break;
7344         case C1LS:
7345           c1ls_alloc(&current,i);
7346           break;
7347         case C2LS:
7348           c2ls_alloc(&current,i);
7349           break;
7350         case C2OP:
7351           c2op_alloc(&current,i);
7352           break;
7353         case SYSCALL:
7354         case HLECALL:
7355         case INTCALL:
7356           syscall_alloc(&current,i);
7357           break;
7358         case SPAN:
7359           pagespan_alloc(&current,i);
7360           break;
7361       }
7362
7363       // Create entry (branch target) regmap
7364       for(hr=0;hr<HOST_REGS;hr++)
7365       {
7366         int r,or;
7367         r=current.regmap[hr];
7368         if(r>=0) {
7369           if(r!=regmap_pre[i][hr]) {
7370             // TODO: delay slot (?)
7371             or=get_reg(regmap_pre[i],r); // Get old mapping for this register
7372             if(or<0||(r&63)>=TEMPREG){
7373               regs[i].regmap_entry[hr]=-1;
7374             }
7375             else
7376             {
7377               // Just move it to a different register
7378               regs[i].regmap_entry[hr]=r;
7379               // If it was dirty before, it's still dirty
7380               if((regs[i].wasdirty>>or)&1) dirty_reg(&current,r&63);
7381             }
7382           }
7383           else
7384           {
7385             // Unneeded
7386             if(r==0){
7387               regs[i].regmap_entry[hr]=0;
7388             }
7389             else
7390             if(r<64){
7391               if((current.u>>r)&1) {
7392                 regs[i].regmap_entry[hr]=-1;
7393                 //regs[i].regmap[hr]=-1;
7394                 current.regmap[hr]=-1;
7395               }else
7396                 regs[i].regmap_entry[hr]=r;
7397             }
7398             else {
7399               assert(0);
7400             }
7401           }
7402         } else {
7403           // Branches expect CCREG to be allocated at the target
7404           if(regmap_pre[i][hr]==CCREG)
7405             regs[i].regmap_entry[hr]=CCREG;
7406           else
7407             regs[i].regmap_entry[hr]=-1;
7408         }
7409       }
7410       memcpy(regs[i].regmap,current.regmap,sizeof(current.regmap));
7411     }
7412
7413     if(i>0&&(itype[i-1]==STORE||itype[i-1]==STORELR||(itype[i-1]==C2LS&&opcode[i-1]==0x3a))&&(u_int)imm[i-1]<0x800)
7414       current.waswritten|=1<<rs1[i-1];
7415     current.waswritten&=~(1<<rt1[i]);
7416     current.waswritten&=~(1<<rt2[i]);
7417     if((itype[i]==STORE||itype[i]==STORELR||(itype[i]==C2LS&&opcode[i]==0x3a))&&(u_int)imm[i]>=0x800)
7418       current.waswritten&=~(1<<rs1[i]);
7419
7420     /* Branch post-alloc */
7421     if(i>0)
7422     {
7423       current.wasdirty=current.dirty;
7424       switch(itype[i-1]) {
7425         case UJUMP:
7426           memcpy(&branch_regs[i-1],&current,sizeof(current));
7427           branch_regs[i-1].isconst=0;
7428           branch_regs[i-1].wasconst=0;
7429           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7430           alloc_cc(&branch_regs[i-1],i-1);
7431           dirty_reg(&branch_regs[i-1],CCREG);
7432           if(rt1[i-1]==31) { // JAL
7433             alloc_reg(&branch_regs[i-1],i-1,31);
7434             dirty_reg(&branch_regs[i-1],31);
7435           }
7436           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7437           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7438           break;
7439         case RJUMP:
7440           memcpy(&branch_regs[i-1],&current,sizeof(current));
7441           branch_regs[i-1].isconst=0;
7442           branch_regs[i-1].wasconst=0;
7443           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7444           alloc_cc(&branch_regs[i-1],i-1);
7445           dirty_reg(&branch_regs[i-1],CCREG);
7446           alloc_reg(&branch_regs[i-1],i-1,rs1[i-1]);
7447           if(rt1[i-1]!=0) { // JALR
7448             alloc_reg(&branch_regs[i-1],i-1,rt1[i-1]);
7449             dirty_reg(&branch_regs[i-1],rt1[i-1]);
7450           }
7451           #ifdef USE_MINI_HT
7452           if(rs1[i-1]==31) { // JALR
7453             alloc_reg(&branch_regs[i-1],i-1,RHASH);
7454             alloc_reg(&branch_regs[i-1],i-1,RHTBL);
7455           }
7456           #endif
7457           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7458           memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7459           break;
7460         case CJUMP:
7461           if((opcode[i-1]&0x3E)==4) // BEQ/BNE
7462           {
7463             alloc_cc(&current,i-1);
7464             dirty_reg(&current,CCREG);
7465             if((rs1[i-1]&&(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]))||
7466                (rs2[i-1]&&(rs2[i-1]==rt1[i]||rs2[i-1]==rt2[i]))) {
7467               // The delay slot overwrote one of our conditions
7468               // Delay slot goes after the test (in order)
7469               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7470               current.u|=1;
7471               delayslot_alloc(&current,i);
7472               current.isconst=0;
7473             }
7474             else
7475             {
7476               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
7477               // Alloc the branch condition registers
7478               if(rs1[i-1]) alloc_reg(&current,i-1,rs1[i-1]);
7479               if(rs2[i-1]) alloc_reg(&current,i-1,rs2[i-1]);
7480             }
7481             memcpy(&branch_regs[i-1],&current,sizeof(current));
7482             branch_regs[i-1].isconst=0;
7483             branch_regs[i-1].wasconst=0;
7484             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7485             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7486           }
7487           else
7488           if((opcode[i-1]&0x3E)==6) // BLEZ/BGTZ
7489           {
7490             alloc_cc(&current,i-1);
7491             dirty_reg(&current,CCREG);
7492             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7493               // The delay slot overwrote the branch condition
7494               // Delay slot goes after the test (in order)
7495               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7496               current.u|=1;
7497               delayslot_alloc(&current,i);
7498               current.isconst=0;
7499             }
7500             else
7501             {
7502               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7503               // Alloc the branch condition register
7504               alloc_reg(&current,i-1,rs1[i-1]);
7505             }
7506             memcpy(&branch_regs[i-1],&current,sizeof(current));
7507             branch_regs[i-1].isconst=0;
7508             branch_regs[i-1].wasconst=0;
7509             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7510             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7511           }
7512           else
7513           // Alloc the delay slot in case the branch is taken
7514           if((opcode[i-1]&0x3E)==0x14) // BEQL/BNEL
7515           {
7516             memcpy(&branch_regs[i-1],&current,sizeof(current));
7517             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7518             alloc_cc(&branch_regs[i-1],i);
7519             dirty_reg(&branch_regs[i-1],CCREG);
7520             delayslot_alloc(&branch_regs[i-1],i);
7521             branch_regs[i-1].isconst=0;
7522             alloc_reg(&current,i,CCREG); // Not taken path
7523             dirty_reg(&current,CCREG);
7524             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7525           }
7526           else
7527           if((opcode[i-1]&0x3E)==0x16) // BLEZL/BGTZL
7528           {
7529             memcpy(&branch_regs[i-1],&current,sizeof(current));
7530             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7531             alloc_cc(&branch_regs[i-1],i);
7532             dirty_reg(&branch_regs[i-1],CCREG);
7533             delayslot_alloc(&branch_regs[i-1],i);
7534             branch_regs[i-1].isconst=0;
7535             alloc_reg(&current,i,CCREG); // Not taken path
7536             dirty_reg(&current,CCREG);
7537             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7538           }
7539           break;
7540         case SJUMP:
7541           //if((opcode2[i-1]&0x1E)==0) // BLTZ/BGEZ
7542           if((opcode2[i-1]&0x0E)==0) // BLTZ/BGEZ
7543           {
7544             alloc_cc(&current,i-1);
7545             dirty_reg(&current,CCREG);
7546             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
7547               // The delay slot overwrote the branch condition
7548               // Delay slot goes after the test (in order)
7549               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
7550               current.u|=1;
7551               delayslot_alloc(&current,i);
7552               current.isconst=0;
7553             }
7554             else
7555             {
7556               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
7557               // Alloc the branch condition register
7558               alloc_reg(&current,i-1,rs1[i-1]);
7559             }
7560             memcpy(&branch_regs[i-1],&current,sizeof(current));
7561             branch_regs[i-1].isconst=0;
7562             branch_regs[i-1].wasconst=0;
7563             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
7564             memcpy(constmap[i],constmap[i-1],sizeof(current_constmap));
7565           }
7566           else
7567           // Alloc the delay slot in case the branch is taken
7568           if((opcode2[i-1]&0x1E)==2) // BLTZL/BGEZL
7569           {
7570             memcpy(&branch_regs[i-1],&current,sizeof(current));
7571             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
7572             alloc_cc(&branch_regs[i-1],i);
7573             dirty_reg(&branch_regs[i-1],CCREG);
7574             delayslot_alloc(&branch_regs[i-1],i);
7575             branch_regs[i-1].isconst=0;
7576             alloc_reg(&current,i,CCREG); // Not taken path
7577             dirty_reg(&current,CCREG);
7578             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
7579           }
7580           // FIXME: BLTZAL/BGEZAL
7581           if(opcode2[i-1]&0x10) { // BxxZAL
7582             alloc_reg(&branch_regs[i-1],i-1,31);
7583             dirty_reg(&branch_regs[i-1],31);
7584           }
7585           break;
7586       }
7587
7588       if(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)
7589       {
7590         if(rt1[i-1]==31) // JAL/JALR
7591         {
7592           // Subroutine call will return here, don't alloc any registers
7593           current.dirty=0;
7594           clear_all_regs(current.regmap);
7595           alloc_reg(&current,i,CCREG);
7596           dirty_reg(&current,CCREG);
7597         }
7598         else if(i+1<slen)
7599         {
7600           // Internal branch will jump here, match registers to caller
7601           current.dirty=0;
7602           clear_all_regs(current.regmap);
7603           alloc_reg(&current,i,CCREG);
7604           dirty_reg(&current,CCREG);
7605           for(j=i-1;j>=0;j--)
7606           {
7607             if(ba[j]==start+i*4+4) {
7608               memcpy(current.regmap,branch_regs[j].regmap,sizeof(current.regmap));
7609               current.dirty=branch_regs[j].dirty;
7610               break;
7611             }
7612           }
7613           while(j>=0) {
7614             if(ba[j]==start+i*4+4) {
7615               for(hr=0;hr<HOST_REGS;hr++) {
7616                 if(current.regmap[hr]!=branch_regs[j].regmap[hr]) {
7617                   current.regmap[hr]=-1;
7618                 }
7619                 current.dirty&=branch_regs[j].dirty;
7620               }
7621             }
7622             j--;
7623           }
7624         }
7625       }
7626     }
7627
7628     // Count cycles in between branches
7629     ccadj[i]=cc;
7630     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i]==SYSCALL||itype[i]==HLECALL))
7631     {
7632       cc=0;
7633     }
7634 #if !defined(DRC_DBG)
7635     else if(itype[i]==C2OP&&gte_cycletab[source[i]&0x3f]>2)
7636     {
7637       // GTE runs in parallel until accessed, divide by 2 for a rough guess
7638       cc+=gte_cycletab[source[i]&0x3f]/2;
7639     }
7640     else if(/*itype[i]==LOAD||itype[i]==STORE||*/itype[i]==C1LS) // load,store causes weird timing issues
7641     {
7642       cc+=2; // 2 cycle penalty (after CLOCK_DIVIDER)
7643     }
7644     else if(i>1&&itype[i]==STORE&&itype[i-1]==STORE&&itype[i-2]==STORE&&!bt[i])
7645     {
7646       cc+=4;
7647     }
7648     else if(itype[i]==C2LS)
7649     {
7650       cc+=4;
7651     }
7652 #endif
7653     else
7654     {
7655       cc++;
7656     }
7657
7658     if(!is_ds[i]) {
7659       regs[i].dirty=current.dirty;
7660       regs[i].isconst=current.isconst;
7661       memcpy(constmap[i],current_constmap,sizeof(current_constmap));
7662     }
7663     for(hr=0;hr<HOST_REGS;hr++) {
7664       if(hr!=EXCLUDE_REG&&regs[i].regmap[hr]>=0) {
7665         if(regmap_pre[i][hr]!=regs[i].regmap[hr]) {
7666           regs[i].wasconst&=~(1<<hr);
7667         }
7668       }
7669     }
7670     if(current.regmap[HOST_BTREG]==BTREG) current.regmap[HOST_BTREG]=-1;
7671     regs[i].waswritten=current.waswritten;
7672   }
7673
7674   /* Pass 4 - Cull unused host registers */
7675
7676   uint64_t nr=0;
7677
7678   for (i=slen-1;i>=0;i--)
7679   {
7680     int hr;
7681     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7682     {
7683       if(ba[i]<start || ba[i]>=(start+slen*4))
7684       {
7685         // Branch out of this block, don't need anything
7686         nr=0;
7687       }
7688       else
7689       {
7690         // Internal branch
7691         // Need whatever matches the target
7692         nr=0;
7693         int t=(ba[i]-start)>>2;
7694         for(hr=0;hr<HOST_REGS;hr++)
7695         {
7696           if(regs[i].regmap_entry[hr]>=0) {
7697             if(regs[i].regmap_entry[hr]==regs[t].regmap_entry[hr]) nr|=1<<hr;
7698           }
7699         }
7700       }
7701       // Conditional branch may need registers for following instructions
7702       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7703       {
7704         if(i<slen-2) {
7705           nr|=needed_reg[i+2];
7706           for(hr=0;hr<HOST_REGS;hr++)
7707           {
7708             if(regmap_pre[i+2][hr]>=0&&get_reg(regs[i+2].regmap_entry,regmap_pre[i+2][hr])<0) nr&=~(1<<hr);
7709             //if((regmap_entry[i+2][hr])>=0) if(!((nr>>hr)&1)) printf("%x-bogus(%d=%d)\n",start+i*4,hr,regmap_entry[i+2][hr]);
7710           }
7711         }
7712       }
7713       // Don't need stuff which is overwritten
7714       //if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
7715       //if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
7716       // Merge in delay slot
7717       for(hr=0;hr<HOST_REGS;hr++)
7718       {
7719         if(!likely[i]) {
7720           // These are overwritten unless the branch is "likely"
7721           // and the delay slot is nullified if not taken
7722           if(rt1[i+1]&&rt1[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7723           if(rt2[i+1]&&rt2[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7724         }
7725         if(us1[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7726         if(us2[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7727         if(rs1[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7728         if(rs2[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
7729         if(us1[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7730         if(us2[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7731         if(rs1[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7732         if(rs2[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7733         if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) {
7734           if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
7735           if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
7736         }
7737       }
7738     }
7739     else if(itype[i]==SYSCALL||itype[i]==HLECALL||itype[i]==INTCALL)
7740     {
7741       // SYSCALL instruction (software interrupt)
7742       nr=0;
7743     }
7744     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
7745     {
7746       // ERET instruction (return from interrupt)
7747       nr=0;
7748     }
7749     else // Non-branch
7750     {
7751       if(i<slen-1) {
7752         for(hr=0;hr<HOST_REGS;hr++) {
7753           if(regmap_pre[i+1][hr]>=0&&get_reg(regs[i+1].regmap_entry,regmap_pre[i+1][hr])<0) nr&=~(1<<hr);
7754           if(regs[i].regmap[hr]!=regmap_pre[i+1][hr]) nr&=~(1<<hr);
7755           if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
7756           if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
7757         }
7758       }
7759     }
7760     for(hr=0;hr<HOST_REGS;hr++)
7761     {
7762       // Overwritten registers are not needed
7763       if(rt1[i]&&rt1[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7764       if(rt2[i]&&rt2[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7765       if(FTEMP==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
7766       // Source registers are needed
7767       if(us1[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7768       if(us2[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7769       if(rs1[i]==regmap_pre[i][hr]) nr|=1<<hr;
7770       if(rs2[i]==regmap_pre[i][hr]) nr|=1<<hr;
7771       if(us1[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7772       if(us2[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7773       if(rs1[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7774       if(rs2[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
7775       if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) {
7776         if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
7777         if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
7778       }
7779       // Don't store a register immediately after writing it,
7780       // may prevent dual-issue.
7781       // But do so if this is a branch target, otherwise we
7782       // might have to load the register before the branch.
7783       if(i>0&&!bt[i]&&((regs[i].wasdirty>>hr)&1)) {
7784         if((regmap_pre[i][hr]>0&&regmap_pre[i][hr]<64&&!((unneeded_reg[i]>>regmap_pre[i][hr])&1))) {
7785           if(rt1[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7786           if(rt2[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
7787         }
7788         if((regs[i].regmap_entry[hr]>0&&regs[i].regmap_entry[hr]<64&&!((unneeded_reg[i]>>regs[i].regmap_entry[hr])&1))) {
7789           if(rt1[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7790           if(rt2[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
7791         }
7792       }
7793     }
7794     // Cycle count is needed at branches.  Assume it is needed at the target too.
7795     if(i==0||bt[i]||itype[i]==CJUMP||itype[i]==SPAN) {
7796       if(regmap_pre[i][HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
7797       if(regs[i].regmap_entry[HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
7798     }
7799     // Save it
7800     needed_reg[i]=nr;
7801
7802     // Deallocate unneeded registers
7803     for(hr=0;hr<HOST_REGS;hr++)
7804     {
7805       if(!((nr>>hr)&1)) {
7806         if(regs[i].regmap_entry[hr]!=CCREG) regs[i].regmap_entry[hr]=-1;
7807         if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
7808            (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7809            (regs[i].regmap[hr]&63)!=PTEMP && (regs[i].regmap[hr]&63)!=CCREG)
7810         {
7811           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7812           {
7813             if(likely[i]) {
7814               regs[i].regmap[hr]=-1;
7815               regs[i].isconst&=~(1<<hr);
7816               if(i<slen-2) {
7817                 regmap_pre[i+2][hr]=-1;
7818                 regs[i+2].wasconst&=~(1<<hr);
7819               }
7820             }
7821           }
7822         }
7823         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7824         {
7825           int d1=0,d2=0,map=0,temp=0;
7826           if(get_reg(regs[i].regmap,rt1[i+1]|64)>=0||get_reg(branch_regs[i].regmap,rt1[i+1]|64)>=0)
7827           {
7828             d1=dep1[i+1];
7829             d2=dep2[i+1];
7830           }
7831           if(itype[i+1]==STORE || itype[i+1]==STORELR ||
7832              (opcode[i+1]&0x3b)==0x39 || (opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
7833             map=INVCP;
7834           }
7835           if(itype[i+1]==LOADLR || itype[i+1]==STORELR ||
7836              itype[i+1]==C1LS || itype[i+1]==C2LS)
7837             temp=FTEMP;
7838           if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
7839              (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7840              (regs[i].regmap[hr]&63)!=rt1[i+1] && (regs[i].regmap[hr]&63)!=rt2[i+1] &&
7841              (regs[i].regmap[hr]^64)!=us1[i+1] && (regs[i].regmap[hr]^64)!=us2[i+1] &&
7842              (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
7843              regs[i].regmap[hr]!=rs1[i+1] && regs[i].regmap[hr]!=rs2[i+1] &&
7844              (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=PTEMP &&
7845              regs[i].regmap[hr]!=RHASH && regs[i].regmap[hr]!=RHTBL &&
7846              regs[i].regmap[hr]!=RTEMP && regs[i].regmap[hr]!=CCREG &&
7847              regs[i].regmap[hr]!=map )
7848           {
7849             regs[i].regmap[hr]=-1;
7850             regs[i].isconst&=~(1<<hr);
7851             if((branch_regs[i].regmap[hr]&63)!=rs1[i] && (branch_regs[i].regmap[hr]&63)!=rs2[i] &&
7852                (branch_regs[i].regmap[hr]&63)!=rt1[i] && (branch_regs[i].regmap[hr]&63)!=rt2[i] &&
7853                (branch_regs[i].regmap[hr]&63)!=rt1[i+1] && (branch_regs[i].regmap[hr]&63)!=rt2[i+1] &&
7854                (branch_regs[i].regmap[hr]^64)!=us1[i+1] && (branch_regs[i].regmap[hr]^64)!=us2[i+1] &&
7855                (branch_regs[i].regmap[hr]^64)!=d1 && (branch_regs[i].regmap[hr]^64)!=d2 &&
7856                branch_regs[i].regmap[hr]!=rs1[i+1] && branch_regs[i].regmap[hr]!=rs2[i+1] &&
7857                (branch_regs[i].regmap[hr]&63)!=temp && branch_regs[i].regmap[hr]!=PTEMP &&
7858                branch_regs[i].regmap[hr]!=RHASH && branch_regs[i].regmap[hr]!=RHTBL &&
7859                branch_regs[i].regmap[hr]!=RTEMP && branch_regs[i].regmap[hr]!=CCREG &&
7860                branch_regs[i].regmap[hr]!=map)
7861             {
7862               branch_regs[i].regmap[hr]=-1;
7863               branch_regs[i].regmap_entry[hr]=-1;
7864               if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
7865               {
7866                 if(!likely[i]&&i<slen-2) {
7867                   regmap_pre[i+2][hr]=-1;
7868                   regs[i+2].wasconst&=~(1<<hr);
7869                 }
7870               }
7871             }
7872           }
7873         }
7874         else
7875         {
7876           // Non-branch
7877           if(i>0)
7878           {
7879             int d1=0,d2=0,map=-1,temp=-1;
7880             if(get_reg(regs[i].regmap,rt1[i]|64)>=0)
7881             {
7882               d1=dep1[i];
7883               d2=dep2[i];
7884             }
7885             if(itype[i]==STORE || itype[i]==STORELR ||
7886                       (opcode[i]&0x3b)==0x39 || (opcode[i]&0x3b)==0x3a) { // SWC1/SDC1 || SWC2/SDC2
7887               map=INVCP;
7888             }
7889             if(itype[i]==LOADLR || itype[i]==STORELR ||
7890                itype[i]==C1LS || itype[i]==C2LS)
7891               temp=FTEMP;
7892             if((regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
7893                (regs[i].regmap[hr]^64)!=us1[i] && (regs[i].regmap[hr]^64)!=us2[i] &&
7894                (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
7895                regs[i].regmap[hr]!=rs1[i] && regs[i].regmap[hr]!=rs2[i] &&
7896                (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=map &&
7897                (itype[i]!=SPAN||regs[i].regmap[hr]!=CCREG))
7898             {
7899               if(i<slen-1&&!is_ds[i]) {
7900                 assert(regs[i].regmap[hr]<64);
7901                 if(regmap_pre[i+1][hr]!=-1 || regs[i].regmap[hr]!=-1)
7902                 if(regmap_pre[i+1][hr]!=regs[i].regmap[hr])
7903                 {
7904                   SysPrintf("fail: %x (%d %d!=%d)\n",start+i*4,hr,regmap_pre[i+1][hr],regs[i].regmap[hr]);
7905                   assert(regmap_pre[i+1][hr]==regs[i].regmap[hr]);
7906                 }
7907                 regmap_pre[i+1][hr]=-1;
7908                 if(regs[i+1].regmap_entry[hr]==CCREG) regs[i+1].regmap_entry[hr]=-1;
7909                 regs[i+1].wasconst&=~(1<<hr);
7910               }
7911               regs[i].regmap[hr]=-1;
7912               regs[i].isconst&=~(1<<hr);
7913             }
7914           }
7915         }
7916       }
7917     }
7918   }
7919
7920   /* Pass 5 - Pre-allocate registers */
7921
7922   // If a register is allocated during a loop, try to allocate it for the
7923   // entire loop, if possible.  This avoids loading/storing registers
7924   // inside of the loop.
7925
7926   signed char f_regmap[HOST_REGS];
7927   clear_all_regs(f_regmap);
7928   for(i=0;i<slen-1;i++)
7929   {
7930     if(itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
7931     {
7932       if(ba[i]>=start && ba[i]<(start+i*4))
7933       if(itype[i+1]==NOP||itype[i+1]==MOV||itype[i+1]==ALU
7934       ||itype[i+1]==SHIFTIMM||itype[i+1]==IMM16||itype[i+1]==LOAD
7935       ||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS
7936       ||itype[i+1]==SHIFT||itype[i+1]==COP1
7937       ||itype[i+1]==COP2||itype[i+1]==C2LS||itype[i+1]==C2OP)
7938       {
7939         int t=(ba[i]-start)>>2;
7940         if(t>0&&(itype[t-1]!=UJUMP&&itype[t-1]!=RJUMP&&itype[t-1]!=CJUMP&&itype[t-1]!=SJUMP)) // loop_preload can't handle jumps into delay slots
7941         if(t<2||(itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||rt1[t-2]!=31) // call/ret assumes no registers allocated
7942         for(hr=0;hr<HOST_REGS;hr++)
7943         {
7944           if(regs[i].regmap[hr]>64) {
7945             if(!((regs[i].dirty>>hr)&1))
7946               f_regmap[hr]=regs[i].regmap[hr];
7947             else f_regmap[hr]=-1;
7948           }
7949           else if(regs[i].regmap[hr]>=0) {
7950             if(f_regmap[hr]!=regs[i].regmap[hr]) {
7951               // dealloc old register
7952               int n;
7953               for(n=0;n<HOST_REGS;n++)
7954               {
7955                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
7956               }
7957               // and alloc new one
7958               f_regmap[hr]=regs[i].regmap[hr];
7959             }
7960           }
7961           if(branch_regs[i].regmap[hr]>64) {
7962             if(!((branch_regs[i].dirty>>hr)&1))
7963               f_regmap[hr]=branch_regs[i].regmap[hr];
7964             else f_regmap[hr]=-1;
7965           }
7966           else if(branch_regs[i].regmap[hr]>=0) {
7967             if(f_regmap[hr]!=branch_regs[i].regmap[hr]) {
7968               // dealloc old register
7969               int n;
7970               for(n=0;n<HOST_REGS;n++)
7971               {
7972                 if(f_regmap[n]==branch_regs[i].regmap[hr]) {f_regmap[n]=-1;}
7973               }
7974               // and alloc new one
7975               f_regmap[hr]=branch_regs[i].regmap[hr];
7976             }
7977           }
7978           if(ooo[i]) {
7979             if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1])
7980               f_regmap[hr]=branch_regs[i].regmap[hr];
7981           }else{
7982             if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1])
7983               f_regmap[hr]=branch_regs[i].regmap[hr];
7984           }
7985           // Avoid dirty->clean transition
7986           #ifdef DESTRUCTIVE_WRITEBACK
7987           if(t>0) if(get_reg(regmap_pre[t],f_regmap[hr])>=0) if((regs[t].wasdirty>>get_reg(regmap_pre[t],f_regmap[hr]))&1) f_regmap[hr]=-1;
7988           #endif
7989           // This check is only strictly required in the DESTRUCTIVE_WRITEBACK
7990           // case above, however it's always a good idea.  We can't hoist the
7991           // load if the register was already allocated, so there's no point
7992           // wasting time analyzing most of these cases.  It only "succeeds"
7993           // when the mapping was different and the load can be replaced with
7994           // a mov, which is of negligible benefit.  So such cases are
7995           // skipped below.
7996           if(f_regmap[hr]>0) {
7997             if(regs[t].regmap[hr]==f_regmap[hr]||(regs[t].regmap_entry[hr]<0&&get_reg(regmap_pre[t],f_regmap[hr])<0)) {
7998               int r=f_regmap[hr];
7999               for(j=t;j<=i;j++)
8000               {
8001                 //printf("Test %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8002                 if(r<34&&((unneeded_reg[j]>>r)&1)) break;
8003                 assert(r < 64);
8004                 if(regs[j].regmap[hr]==f_regmap[hr]&&(f_regmap[hr]&63)<TEMPREG) {
8005                   //printf("Hit %x -> %x, %x %d/%d\n",start+i*4,ba[i],start+j*4,hr,r);
8006                   int k;
8007                   if(regs[i].regmap[hr]==-1&&branch_regs[i].regmap[hr]==-1) {
8008                     if(get_reg(regs[i+2].regmap,f_regmap[hr])>=0) break;
8009                     if(r>63) {
8010                       if(get_reg(regs[i].regmap,r&63)<0) break;
8011                       if(get_reg(branch_regs[i].regmap,r&63)<0) break;
8012                     }
8013                     k=i;
8014                     while(k>1&&regs[k-1].regmap[hr]==-1) {
8015                       if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8016                         //printf("no free regs for store %x\n",start+(k-1)*4);
8017                         break;
8018                       }
8019                       if(get_reg(regs[k-1].regmap,f_regmap[hr])>=0) {
8020                         //printf("no-match due to different register\n");
8021                         break;
8022                       }
8023                       if(itype[k-2]==UJUMP||itype[k-2]==RJUMP||itype[k-2]==CJUMP||itype[k-2]==SJUMP) {
8024                         //printf("no-match due to branch\n");
8025                         break;
8026                       }
8027                       // call/ret fast path assumes no registers allocated
8028                       if(k>2&&(itype[k-3]==UJUMP||itype[k-3]==RJUMP)&&rt1[k-3]==31) {
8029                         break;
8030                       }
8031                       assert(r < 64);
8032                       k--;
8033                     }
8034                     if(regs[k-1].regmap[hr]==f_regmap[hr]&&regmap_pre[k][hr]==f_regmap[hr]) {
8035                       //printf("Extend r%d, %x ->\n",hr,start+k*4);
8036                       while(k<i) {
8037                         regs[k].regmap_entry[hr]=f_regmap[hr];
8038                         regs[k].regmap[hr]=f_regmap[hr];
8039                         regmap_pre[k+1][hr]=f_regmap[hr];
8040                         regs[k].wasdirty&=~(1<<hr);
8041                         regs[k].dirty&=~(1<<hr);
8042                         regs[k].wasdirty|=(1<<hr)&regs[k-1].dirty;
8043                         regs[k].dirty|=(1<<hr)&regs[k].wasdirty;
8044                         regs[k].wasconst&=~(1<<hr);
8045                         regs[k].isconst&=~(1<<hr);
8046                         k++;
8047                       }
8048                     }
8049                     else {
8050                       //printf("Fail Extend r%d, %x ->\n",hr,start+k*4);
8051                       break;
8052                     }
8053                     assert(regs[i-1].regmap[hr]==f_regmap[hr]);
8054                     if(regs[i-1].regmap[hr]==f_regmap[hr]&&regmap_pre[i][hr]==f_regmap[hr]) {
8055                       //printf("OK fill %x (r%d)\n",start+i*4,hr);
8056                       regs[i].regmap_entry[hr]=f_regmap[hr];
8057                       regs[i].regmap[hr]=f_regmap[hr];
8058                       regs[i].wasdirty&=~(1<<hr);
8059                       regs[i].dirty&=~(1<<hr);
8060                       regs[i].wasdirty|=(1<<hr)&regs[i-1].dirty;
8061                       regs[i].dirty|=(1<<hr)&regs[i-1].dirty;
8062                       regs[i].wasconst&=~(1<<hr);
8063                       regs[i].isconst&=~(1<<hr);
8064                       branch_regs[i].regmap_entry[hr]=f_regmap[hr];
8065                       branch_regs[i].wasdirty&=~(1<<hr);
8066                       branch_regs[i].wasdirty|=(1<<hr)&regs[i].dirty;
8067                       branch_regs[i].regmap[hr]=f_regmap[hr];
8068                       branch_regs[i].dirty&=~(1<<hr);
8069                       branch_regs[i].dirty|=(1<<hr)&regs[i].dirty;
8070                       branch_regs[i].wasconst&=~(1<<hr);
8071                       branch_regs[i].isconst&=~(1<<hr);
8072                       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
8073                         regmap_pre[i+2][hr]=f_regmap[hr];
8074                         regs[i+2].wasdirty&=~(1<<hr);
8075                         regs[i+2].wasdirty|=(1<<hr)&regs[i].dirty;
8076                       }
8077                     }
8078                   }
8079                   for(k=t;k<j;k++) {
8080                     // Alloc register clean at beginning of loop,
8081                     // but may dirty it in pass 6
8082                     regs[k].regmap_entry[hr]=f_regmap[hr];
8083                     regs[k].regmap[hr]=f_regmap[hr];
8084                     regs[k].dirty&=~(1<<hr);
8085                     regs[k].wasconst&=~(1<<hr);
8086                     regs[k].isconst&=~(1<<hr);
8087                     if(itype[k]==UJUMP||itype[k]==RJUMP||itype[k]==CJUMP||itype[k]==SJUMP) {
8088                       branch_regs[k].regmap_entry[hr]=f_regmap[hr];
8089                       branch_regs[k].regmap[hr]=f_regmap[hr];
8090                       branch_regs[k].dirty&=~(1<<hr);
8091                       branch_regs[k].wasconst&=~(1<<hr);
8092                       branch_regs[k].isconst&=~(1<<hr);
8093                       if(itype[k]!=RJUMP&&itype[k]!=UJUMP&&(source[k]>>16)!=0x1000) {
8094                         regmap_pre[k+2][hr]=f_regmap[hr];
8095                         regs[k+2].wasdirty&=~(1<<hr);
8096                       }
8097                     }
8098                     else
8099                     {
8100                       regmap_pre[k+1][hr]=f_regmap[hr];
8101                       regs[k+1].wasdirty&=~(1<<hr);
8102                     }
8103                   }
8104                   if(regs[j].regmap[hr]==f_regmap[hr])
8105                     regs[j].regmap_entry[hr]=f_regmap[hr];
8106                   break;
8107                 }
8108                 if(j==i) break;
8109                 if(regs[j].regmap[hr]>=0)
8110                   break;
8111                 if(get_reg(regs[j].regmap,f_regmap[hr])>=0) {
8112                   //printf("no-match due to different register\n");
8113                   break;
8114                 }
8115                 if(itype[j]==UJUMP||itype[j]==RJUMP||(source[j]>>16)==0x1000)
8116                 {
8117                   // Stop on unconditional branch
8118                   break;
8119                 }
8120                 if(itype[j]==CJUMP||itype[j]==SJUMP)
8121                 {
8122                   if(ooo[j]) {
8123                     if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1])
8124                       break;
8125                   }else{
8126                     if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1])
8127                       break;
8128                   }
8129                   if(get_reg(branch_regs[j].regmap,f_regmap[hr])>=0) {
8130                     //printf("no-match due to different register (branch)\n");
8131                     break;
8132                   }
8133                 }
8134                 if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8135                   //printf("No free regs for store %x\n",start+j*4);
8136                   break;
8137                 }
8138                 assert(f_regmap[hr]<64);
8139               }
8140             }
8141           }
8142         }
8143       }
8144     }else{
8145       // Non branch or undetermined branch target
8146       for(hr=0;hr<HOST_REGS;hr++)
8147       {
8148         if(hr!=EXCLUDE_REG) {
8149           if(regs[i].regmap[hr]>64) {
8150             if(!((regs[i].dirty>>hr)&1))
8151               f_regmap[hr]=regs[i].regmap[hr];
8152           }
8153           else if(regs[i].regmap[hr]>=0) {
8154             if(f_regmap[hr]!=regs[i].regmap[hr]) {
8155               // dealloc old register
8156               int n;
8157               for(n=0;n<HOST_REGS;n++)
8158               {
8159                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
8160               }
8161               // and alloc new one
8162               f_regmap[hr]=regs[i].regmap[hr];
8163             }
8164           }
8165         }
8166       }
8167       // Try to restore cycle count at branch targets
8168       if(bt[i]) {
8169         for(j=i;j<slen-1;j++) {
8170           if(regs[j].regmap[HOST_CCREG]!=-1) break;
8171           if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
8172             //printf("no free regs for store %x\n",start+j*4);
8173             break;
8174           }
8175         }
8176         if(regs[j].regmap[HOST_CCREG]==CCREG) {
8177           int k=i;
8178           //printf("Extend CC, %x -> %x\n",start+k*4,start+j*4);
8179           while(k<j) {
8180             regs[k].regmap_entry[HOST_CCREG]=CCREG;
8181             regs[k].regmap[HOST_CCREG]=CCREG;
8182             regmap_pre[k+1][HOST_CCREG]=CCREG;
8183             regs[k+1].wasdirty|=1<<HOST_CCREG;
8184             regs[k].dirty|=1<<HOST_CCREG;
8185             regs[k].wasconst&=~(1<<HOST_CCREG);
8186             regs[k].isconst&=~(1<<HOST_CCREG);
8187             k++;
8188           }
8189           regs[j].regmap_entry[HOST_CCREG]=CCREG;
8190         }
8191         // Work backwards from the branch target
8192         if(j>i&&f_regmap[HOST_CCREG]==CCREG)
8193         {
8194           //printf("Extend backwards\n");
8195           int k;
8196           k=i;
8197           while(regs[k-1].regmap[HOST_CCREG]==-1) {
8198             if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
8199               //printf("no free regs for store %x\n",start+(k-1)*4);
8200               break;
8201             }
8202             k--;
8203           }
8204           if(regs[k-1].regmap[HOST_CCREG]==CCREG) {
8205             //printf("Extend CC, %x ->\n",start+k*4);
8206             while(k<=i) {
8207               regs[k].regmap_entry[HOST_CCREG]=CCREG;
8208               regs[k].regmap[HOST_CCREG]=CCREG;
8209               regmap_pre[k+1][HOST_CCREG]=CCREG;
8210               regs[k+1].wasdirty|=1<<HOST_CCREG;
8211               regs[k].dirty|=1<<HOST_CCREG;
8212               regs[k].wasconst&=~(1<<HOST_CCREG);
8213               regs[k].isconst&=~(1<<HOST_CCREG);
8214               k++;
8215             }
8216           }
8217           else {
8218             //printf("Fail Extend CC, %x ->\n",start+k*4);
8219           }
8220         }
8221       }
8222       if(itype[i]!=STORE&&itype[i]!=STORELR&&itype[i]!=C1LS&&itype[i]!=SHIFT&&
8223          itype[i]!=NOP&&itype[i]!=MOV&&itype[i]!=ALU&&itype[i]!=SHIFTIMM&&
8224          itype[i]!=IMM16&&itype[i]!=LOAD&&itype[i]!=COP1)
8225       {
8226         memcpy(f_regmap,regs[i].regmap,sizeof(f_regmap));
8227       }
8228     }
8229   }
8230
8231   // This allocates registers (if possible) one instruction prior
8232   // to use, which can avoid a load-use penalty on certain CPUs.
8233   for(i=0;i<slen-1;i++)
8234   {
8235     if(!i||(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP))
8236     {
8237       if(!bt[i+1])
8238       {
8239         if(itype[i]==ALU||itype[i]==MOV||itype[i]==LOAD||itype[i]==SHIFTIMM||itype[i]==IMM16
8240            ||((itype[i]==COP1||itype[i]==COP2)&&opcode2[i]<3))
8241         {
8242           if(rs1[i+1]) {
8243             if((hr=get_reg(regs[i+1].regmap,rs1[i+1]))>=0)
8244             {
8245               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8246               {
8247                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8248                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8249                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8250                 regs[i].isconst&=~(1<<hr);
8251                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8252                 constmap[i][hr]=constmap[i+1][hr];
8253                 regs[i+1].wasdirty&=~(1<<hr);
8254                 regs[i].dirty&=~(1<<hr);
8255               }
8256             }
8257           }
8258           if(rs2[i+1]) {
8259             if((hr=get_reg(regs[i+1].regmap,rs2[i+1]))>=0)
8260             {
8261               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8262               {
8263                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
8264                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
8265                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
8266                 regs[i].isconst&=~(1<<hr);
8267                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8268                 constmap[i][hr]=constmap[i+1][hr];
8269                 regs[i+1].wasdirty&=~(1<<hr);
8270                 regs[i].dirty&=~(1<<hr);
8271               }
8272             }
8273           }
8274           // Preload target address for load instruction (non-constant)
8275           if(itype[i+1]==LOAD&&rs1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8276             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8277             {
8278               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8279               {
8280                 regs[i].regmap[hr]=rs1[i+1];
8281                 regmap_pre[i+1][hr]=rs1[i+1];
8282                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8283                 regs[i].isconst&=~(1<<hr);
8284                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8285                 constmap[i][hr]=constmap[i+1][hr];
8286                 regs[i+1].wasdirty&=~(1<<hr);
8287                 regs[i].dirty&=~(1<<hr);
8288               }
8289             }
8290           }
8291           // Load source into target register
8292           if(lt1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8293             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
8294             {
8295               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8296               {
8297                 regs[i].regmap[hr]=rs1[i+1];
8298                 regmap_pre[i+1][hr]=rs1[i+1];
8299                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8300                 regs[i].isconst&=~(1<<hr);
8301                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8302                 constmap[i][hr]=constmap[i+1][hr];
8303                 regs[i+1].wasdirty&=~(1<<hr);
8304                 regs[i].dirty&=~(1<<hr);
8305               }
8306             }
8307           }
8308           // Address for store instruction (non-constant)
8309           if(itype[i+1]==STORE||itype[i+1]==STORELR
8310              ||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SB/SH/SW/SD/SWC1/SDC1/SWC2/SDC2
8311             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8312               hr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1);
8313               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8314               else {regs[i+1].regmap[hr]=AGEN1+((i+1)&1);regs[i+1].isconst&=~(1<<hr);}
8315               assert(hr>=0);
8316               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8317               {
8318                 regs[i].regmap[hr]=rs1[i+1];
8319                 regmap_pre[i+1][hr]=rs1[i+1];
8320                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8321                 regs[i].isconst&=~(1<<hr);
8322                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8323                 constmap[i][hr]=constmap[i+1][hr];
8324                 regs[i+1].wasdirty&=~(1<<hr);
8325                 regs[i].dirty&=~(1<<hr);
8326               }
8327             }
8328           }
8329           if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) { // LWC1/LDC1, LWC2/LDC2
8330             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
8331               int nr;
8332               hr=get_reg(regs[i+1].regmap,FTEMP);
8333               assert(hr>=0);
8334               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
8335               {
8336                 regs[i].regmap[hr]=rs1[i+1];
8337                 regmap_pre[i+1][hr]=rs1[i+1];
8338                 regs[i+1].regmap_entry[hr]=rs1[i+1];
8339                 regs[i].isconst&=~(1<<hr);
8340                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
8341                 constmap[i][hr]=constmap[i+1][hr];
8342                 regs[i+1].wasdirty&=~(1<<hr);
8343                 regs[i].dirty&=~(1<<hr);
8344               }
8345               else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
8346               {
8347                 // move it to another register
8348                 regs[i+1].regmap[hr]=-1;
8349                 regmap_pre[i+2][hr]=-1;
8350                 regs[i+1].regmap[nr]=FTEMP;
8351                 regmap_pre[i+2][nr]=FTEMP;
8352                 regs[i].regmap[nr]=rs1[i+1];
8353                 regmap_pre[i+1][nr]=rs1[i+1];
8354                 regs[i+1].regmap_entry[nr]=rs1[i+1];
8355                 regs[i].isconst&=~(1<<nr);
8356                 regs[i+1].isconst&=~(1<<nr);
8357                 regs[i].dirty&=~(1<<nr);
8358                 regs[i+1].wasdirty&=~(1<<nr);
8359                 regs[i+1].dirty&=~(1<<nr);
8360                 regs[i+2].wasdirty&=~(1<<nr);
8361               }
8362             }
8363           }
8364           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR/*||itype[i+1]==C1LS||||itype[i+1]==C2LS*/) {
8365             if(itype[i+1]==LOAD)
8366               hr=get_reg(regs[i+1].regmap,rt1[i+1]);
8367             if(itype[i+1]==LOADLR||(opcode[i+1]&0x3b)==0x31||(opcode[i+1]&0x3b)==0x32) // LWC1/LDC1, LWC2/LDC2
8368               hr=get_reg(regs[i+1].regmap,FTEMP);
8369             if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a) { // SWC1/SDC1/SWC2/SDC2
8370               hr=get_reg(regs[i+1].regmap,AGEN1+((i+1)&1));
8371               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
8372             }
8373             if(hr>=0&&regs[i].regmap[hr]<0) {
8374               int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
8375               if(rs>=0&&((regs[i+1].wasconst>>rs)&1)) {
8376                 regs[i].regmap[hr]=AGEN1+((i+1)&1);
8377                 regmap_pre[i+1][hr]=AGEN1+((i+1)&1);
8378                 regs[i+1].regmap_entry[hr]=AGEN1+((i+1)&1);
8379                 regs[i].isconst&=~(1<<hr);
8380                 regs[i+1].wasdirty&=~(1<<hr);
8381                 regs[i].dirty&=~(1<<hr);
8382               }
8383             }
8384           }
8385         }
8386       }
8387     }
8388   }
8389
8390   /* Pass 6 - Optimize clean/dirty state */
8391   clean_registers(0,slen-1,1);
8392
8393   /* Pass 7 - Identify 32-bit registers */
8394   for (i=slen-1;i>=0;i--)
8395   {
8396     if(itype[i]==CJUMP||itype[i]==SJUMP)
8397     {
8398       // Conditional branch
8399       if((source[i]>>16)!=0x1000&&i<slen-2) {
8400         // Mark this address as a branch target since it may be called
8401         // upon return from interrupt
8402         bt[i+2]=1;
8403       }
8404     }
8405   }
8406
8407   if(itype[slen-1]==SPAN) {
8408     bt[slen-1]=1; // Mark as a branch target so instruction can restart after exception
8409   }
8410
8411 #ifdef DISASM
8412   /* Debug/disassembly */
8413   for(i=0;i<slen;i++)
8414   {
8415     printf("U:");
8416     int r;
8417     for(r=1;r<=CCREG;r++) {
8418       if((unneeded_reg[i]>>r)&1) {
8419         if(r==HIREG) printf(" HI");
8420         else if(r==LOREG) printf(" LO");
8421         else printf(" r%d",r);
8422       }
8423     }
8424     printf("\n");
8425     #if defined(__i386__) || defined(__x86_64__)
8426     printf("pre: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7]);
8427     #endif
8428     #ifdef __arm__
8429     printf("pre: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][4],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7],regmap_pre[i][8],regmap_pre[i][9],regmap_pre[i][10],regmap_pre[i][12]);
8430     #endif
8431     printf("needs: ");
8432     if(needed_reg[i]&1) printf("eax ");
8433     if((needed_reg[i]>>1)&1) printf("ecx ");
8434     if((needed_reg[i]>>2)&1) printf("edx ");
8435     if((needed_reg[i]>>3)&1) printf("ebx ");
8436     if((needed_reg[i]>>5)&1) printf("ebp ");
8437     if((needed_reg[i]>>6)&1) printf("esi ");
8438     if((needed_reg[i]>>7)&1) printf("edi ");
8439     printf("\n");
8440     #if defined(__i386__) || defined(__x86_64__)
8441     printf("entry: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7]);
8442     printf("dirty: ");
8443     if(regs[i].wasdirty&1) printf("eax ");
8444     if((regs[i].wasdirty>>1)&1) printf("ecx ");
8445     if((regs[i].wasdirty>>2)&1) printf("edx ");
8446     if((regs[i].wasdirty>>3)&1) printf("ebx ");
8447     if((regs[i].wasdirty>>5)&1) printf("ebp ");
8448     if((regs[i].wasdirty>>6)&1) printf("esi ");
8449     if((regs[i].wasdirty>>7)&1) printf("edi ");
8450     #endif
8451     #ifdef __arm__
8452     printf("entry: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d\n",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[4],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7],regs[i].regmap_entry[8],regs[i].regmap_entry[9],regs[i].regmap_entry[10],regs[i].regmap_entry[12]);
8453     printf("dirty: ");
8454     if(regs[i].wasdirty&1) printf("r0 ");
8455     if((regs[i].wasdirty>>1)&1) printf("r1 ");
8456     if((regs[i].wasdirty>>2)&1) printf("r2 ");
8457     if((regs[i].wasdirty>>3)&1) printf("r3 ");
8458     if((regs[i].wasdirty>>4)&1) printf("r4 ");
8459     if((regs[i].wasdirty>>5)&1) printf("r5 ");
8460     if((regs[i].wasdirty>>6)&1) printf("r6 ");
8461     if((regs[i].wasdirty>>7)&1) printf("r7 ");
8462     if((regs[i].wasdirty>>8)&1) printf("r8 ");
8463     if((regs[i].wasdirty>>9)&1) printf("r9 ");
8464     if((regs[i].wasdirty>>10)&1) printf("r10 ");
8465     if((regs[i].wasdirty>>12)&1) printf("r12 ");
8466     #endif
8467     printf("\n");
8468     disassemble_inst(i);
8469     //printf ("ccadj[%d] = %d\n",i,ccadj[i]);
8470     #if defined(__i386__) || defined(__x86_64__)
8471     printf("eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7]);
8472     if(regs[i].dirty&1) printf("eax ");
8473     if((regs[i].dirty>>1)&1) printf("ecx ");
8474     if((regs[i].dirty>>2)&1) printf("edx ");
8475     if((regs[i].dirty>>3)&1) printf("ebx ");
8476     if((regs[i].dirty>>5)&1) printf("ebp ");
8477     if((regs[i].dirty>>6)&1) printf("esi ");
8478     if((regs[i].dirty>>7)&1) printf("edi ");
8479     #endif
8480     #ifdef __arm__
8481     printf("r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[4],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7],regs[i].regmap[8],regs[i].regmap[9],regs[i].regmap[10],regs[i].regmap[12]);
8482     if(regs[i].dirty&1) printf("r0 ");
8483     if((regs[i].dirty>>1)&1) printf("r1 ");
8484     if((regs[i].dirty>>2)&1) printf("r2 ");
8485     if((regs[i].dirty>>3)&1) printf("r3 ");
8486     if((regs[i].dirty>>4)&1) printf("r4 ");
8487     if((regs[i].dirty>>5)&1) printf("r5 ");
8488     if((regs[i].dirty>>6)&1) printf("r6 ");
8489     if((regs[i].dirty>>7)&1) printf("r7 ");
8490     if((regs[i].dirty>>8)&1) printf("r8 ");
8491     if((regs[i].dirty>>9)&1) printf("r9 ");
8492     if((regs[i].dirty>>10)&1) printf("r10 ");
8493     if((regs[i].dirty>>12)&1) printf("r12 ");
8494     #endif
8495     printf("\n");
8496     if(regs[i].isconst) {
8497       printf("constants: ");
8498       #if defined(__i386__) || defined(__x86_64__)
8499       if(regs[i].isconst&1) printf("eax=%x ",(u_int)constmap[i][0]);
8500       if((regs[i].isconst>>1)&1) printf("ecx=%x ",(u_int)constmap[i][1]);
8501       if((regs[i].isconst>>2)&1) printf("edx=%x ",(u_int)constmap[i][2]);
8502       if((regs[i].isconst>>3)&1) printf("ebx=%x ",(u_int)constmap[i][3]);
8503       if((regs[i].isconst>>5)&1) printf("ebp=%x ",(u_int)constmap[i][5]);
8504       if((regs[i].isconst>>6)&1) printf("esi=%x ",(u_int)constmap[i][6]);
8505       if((regs[i].isconst>>7)&1) printf("edi=%x ",(u_int)constmap[i][7]);
8506       #endif
8507       #ifdef __arm__
8508       int r;
8509       for (r = 0; r < ARRAY_SIZE(constmap[i]); r++)
8510         if ((regs[i].isconst >> r) & 1)
8511           printf(" r%d=%x", r, (u_int)constmap[i][r]);
8512       #endif
8513       printf("\n");
8514     }
8515     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
8516       #if defined(__i386__) || defined(__x86_64__)
8517       printf("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
8518       if(branch_regs[i].dirty&1) printf("eax ");
8519       if((branch_regs[i].dirty>>1)&1) printf("ecx ");
8520       if((branch_regs[i].dirty>>2)&1) printf("edx ");
8521       if((branch_regs[i].dirty>>3)&1) printf("ebx ");
8522       if((branch_regs[i].dirty>>5)&1) printf("ebp ");
8523       if((branch_regs[i].dirty>>6)&1) printf("esi ");
8524       if((branch_regs[i].dirty>>7)&1) printf("edi ");
8525       #endif
8526       #ifdef __arm__
8527       printf("branch(%d): r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[4],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7],branch_regs[i].regmap[8],branch_regs[i].regmap[9],branch_regs[i].regmap[10],branch_regs[i].regmap[12]);
8528       if(branch_regs[i].dirty&1) printf("r0 ");
8529       if((branch_regs[i].dirty>>1)&1) printf("r1 ");
8530       if((branch_regs[i].dirty>>2)&1) printf("r2 ");
8531       if((branch_regs[i].dirty>>3)&1) printf("r3 ");
8532       if((branch_regs[i].dirty>>4)&1) printf("r4 ");
8533       if((branch_regs[i].dirty>>5)&1) printf("r5 ");
8534       if((branch_regs[i].dirty>>6)&1) printf("r6 ");
8535       if((branch_regs[i].dirty>>7)&1) printf("r7 ");
8536       if((branch_regs[i].dirty>>8)&1) printf("r8 ");
8537       if((branch_regs[i].dirty>>9)&1) printf("r9 ");
8538       if((branch_regs[i].dirty>>10)&1) printf("r10 ");
8539       if((branch_regs[i].dirty>>12)&1) printf("r12 ");
8540       #endif
8541     }
8542   }
8543 #endif // DISASM
8544
8545   /* Pass 8 - Assembly */
8546   linkcount=0;stubcount=0;
8547   ds=0;is_delayslot=0;
8548   u_int dirty_pre=0;
8549   void *beginning=start_block();
8550   if((u_int)addr&1) {
8551     ds=1;
8552     pagespan_ds();
8553   }
8554   void *instr_addr0_override = NULL;
8555
8556   if (start == 0x80030000) {
8557     // nasty hack for fastbios thing
8558     // override block entry to this code
8559     instr_addr0_override = out;
8560     emit_movimm(start,0);
8561     // abuse io address var as a flag that we
8562     // have already returned here once
8563     emit_readword(&address,1);
8564     emit_writeword(0,&pcaddr);
8565     emit_writeword(0,&address);
8566     emit_cmp(0,1);
8567     emit_jne(new_dyna_leave);
8568   }
8569   for(i=0;i<slen;i++)
8570   {
8571     //if(ds) printf("ds: ");
8572     disassemble_inst(i);
8573     if(ds) {
8574       ds=0; // Skip delay slot
8575       if(bt[i]) assem_debug("OOPS - branch into delay slot\n");
8576       instr_addr[i] = NULL;
8577     } else {
8578       speculate_register_values(i);
8579       #ifndef DESTRUCTIVE_WRITEBACK
8580       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8581       {
8582         wb_valid(regmap_pre[i],regs[i].regmap_entry,dirty_pre,regs[i].wasdirty,unneeded_reg[i]);
8583       }
8584       if((itype[i]==CJUMP||itype[i]==SJUMP)&&!likely[i]) {
8585         dirty_pre=branch_regs[i].dirty;
8586       }else{
8587         dirty_pre=regs[i].dirty;
8588       }
8589       #endif
8590       // write back
8591       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
8592       {
8593         wb_invalidate(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,unneeded_reg[i]);
8594         loop_preload(regmap_pre[i],regs[i].regmap_entry);
8595       }
8596       // branch target entry point
8597       instr_addr[i] = out;
8598       assem_debug("<->\n");
8599       drc_dbg_emit_do_cmp(i);
8600
8601       // load regs
8602       if(regs[i].regmap_entry[HOST_CCREG]==CCREG&&regs[i].regmap[HOST_CCREG]!=CCREG)
8603         wb_register(CCREG,regs[i].regmap_entry,regs[i].wasdirty);
8604       load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i],rs2[i]);
8605       address_generation(i,&regs[i],regs[i].regmap_entry);
8606       load_consts(regmap_pre[i],regs[i].regmap,i);
8607       if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP)
8608       {
8609         // Load the delay slot registers if necessary
8610         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i]&&(rs1[i+1]!=rt1[i]||rt1[i]==0))
8611           load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8612         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i]&&(rs2[i+1]!=rt1[i]||rt1[i]==0))
8613           load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8614         if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39||(opcode[i+1]&0x3b)==0x3a)
8615           load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8616       }
8617       else if(i+1<slen)
8618       {
8619         // Preload registers for following instruction
8620         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
8621           if(rs1[i+1]!=rt1[i]&&rs1[i+1]!=rt2[i])
8622             load_regs(regs[i].regmap_entry,regs[i].regmap,rs1[i+1],rs1[i+1]);
8623         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
8624           if(rs2[i+1]!=rt1[i]&&rs2[i+1]!=rt2[i])
8625             load_regs(regs[i].regmap_entry,regs[i].regmap,rs2[i+1],rs2[i+1]);
8626       }
8627       // TODO: if(is_ooo(i)) address_generation(i+1);
8628       if(itype[i]==CJUMP)
8629         load_regs(regs[i].regmap_entry,regs[i].regmap,CCREG,CCREG);
8630       if(itype[i]==STORE||itype[i]==STORELR||(opcode[i]&0x3b)==0x39||(opcode[i]&0x3b)==0x3a)
8631         load_regs(regs[i].regmap_entry,regs[i].regmap,INVCP,INVCP);
8632       // assemble
8633       switch(itype[i]) {
8634         case ALU:
8635           alu_assemble(i,&regs[i]);break;
8636         case IMM16:
8637           imm16_assemble(i,&regs[i]);break;
8638         case SHIFT:
8639           shift_assemble(i,&regs[i]);break;
8640         case SHIFTIMM:
8641           shiftimm_assemble(i,&regs[i]);break;
8642         case LOAD:
8643           load_assemble(i,&regs[i]);break;
8644         case LOADLR:
8645           loadlr_assemble(i,&regs[i]);break;
8646         case STORE:
8647           store_assemble(i,&regs[i]);break;
8648         case STORELR:
8649           storelr_assemble(i,&regs[i]);break;
8650         case COP0:
8651           cop0_assemble(i,&regs[i]);break;
8652         case COP1:
8653           cop1_assemble(i,&regs[i]);break;
8654         case C1LS:
8655           c1ls_assemble(i,&regs[i]);break;
8656         case COP2:
8657           cop2_assemble(i,&regs[i]);break;
8658         case C2LS:
8659           c2ls_assemble(i,&regs[i]);break;
8660         case C2OP:
8661           c2op_assemble(i,&regs[i]);break;
8662         case MULTDIV:
8663           multdiv_assemble(i,&regs[i]);break;
8664         case MOV:
8665           mov_assemble(i,&regs[i]);break;
8666         case SYSCALL:
8667           syscall_assemble(i,&regs[i]);break;
8668         case HLECALL:
8669           hlecall_assemble(i,&regs[i]);break;
8670         case INTCALL:
8671           intcall_assemble(i,&regs[i]);break;
8672         case UJUMP:
8673           ujump_assemble(i,&regs[i]);ds=1;break;
8674         case RJUMP:
8675           rjump_assemble(i,&regs[i]);ds=1;break;
8676         case CJUMP:
8677           cjump_assemble(i,&regs[i]);ds=1;break;
8678         case SJUMP:
8679           sjump_assemble(i,&regs[i]);ds=1;break;
8680         case SPAN:
8681           pagespan_assemble(i,&regs[i]);break;
8682       }
8683       if(itype[i]==UJUMP||itype[i]==RJUMP||(source[i]>>16)==0x1000)
8684         literal_pool(1024);
8685       else
8686         literal_pool_jumpover(256);
8687     }
8688   }
8689   //assert(itype[i-2]==UJUMP||itype[i-2]==RJUMP||(source[i-2]>>16)==0x1000);
8690   // If the block did not end with an unconditional branch,
8691   // add a jump to the next instruction.
8692   if(i>1) {
8693     if(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000&&itype[i-1]!=SPAN) {
8694       assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8695       assert(i==slen);
8696       if(itype[i-2]!=CJUMP&&itype[i-2]!=SJUMP) {
8697         store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8698         if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8699           emit_loadreg(CCREG,HOST_CCREG);
8700         emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8701       }
8702       else if(!likely[i-2])
8703       {
8704         store_regs_bt(branch_regs[i-2].regmap,branch_regs[i-2].dirty,start+i*4);
8705         assert(branch_regs[i-2].regmap[HOST_CCREG]==CCREG);
8706       }
8707       else
8708       {
8709         store_regs_bt(regs[i-2].regmap,regs[i-2].dirty,start+i*4);
8710         assert(regs[i-2].regmap[HOST_CCREG]==CCREG);
8711       }
8712       add_to_linker(out,start+i*4,0);
8713       emit_jmp(0);
8714     }
8715   }
8716   else
8717   {
8718     assert(i>0);
8719     assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP);
8720     store_regs_bt(regs[i-1].regmap,regs[i-1].dirty,start+i*4);
8721     if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
8722       emit_loadreg(CCREG,HOST_CCREG);
8723     emit_addimm(HOST_CCREG,CLOCK_ADJUST(ccadj[i-1]+1),HOST_CCREG);
8724     add_to_linker(out,start+i*4,0);
8725     emit_jmp(0);
8726   }
8727
8728   // TODO: delay slot stubs?
8729   // Stubs
8730   for(i=0;i<stubcount;i++)
8731   {
8732     switch(stubs[i].type)
8733     {
8734       case LOADB_STUB:
8735       case LOADH_STUB:
8736       case LOADW_STUB:
8737       case LOADD_STUB:
8738       case LOADBU_STUB:
8739       case LOADHU_STUB:
8740         do_readstub(i);break;
8741       case STOREB_STUB:
8742       case STOREH_STUB:
8743       case STOREW_STUB:
8744       case STORED_STUB:
8745         do_writestub(i);break;
8746       case CC_STUB:
8747         do_ccstub(i);break;
8748       case INVCODE_STUB:
8749         do_invstub(i);break;
8750       case FP_STUB:
8751         do_cop1stub(i);break;
8752       case STORELR_STUB:
8753         do_unalignedwritestub(i);break;
8754     }
8755   }
8756
8757   if (instr_addr0_override)
8758     instr_addr[0] = instr_addr0_override;
8759
8760   /* Pass 9 - Linker */
8761   for(i=0;i<linkcount;i++)
8762   {
8763     assem_debug("%p -> %8x\n",link_addr[i].addr,link_addr[i].target);
8764     literal_pool(64);
8765     if (!link_addr[i].ext)
8766     {
8767       void *stub = out;
8768       void *addr = check_addr(link_addr[i].target);
8769       emit_extjump(link_addr[i].addr, link_addr[i].target);
8770       if (addr) {
8771         set_jump_target(link_addr[i].addr, addr);
8772         add_link(link_addr[i].target,stub);
8773       }
8774       else
8775         set_jump_target(link_addr[i].addr, stub);
8776     }
8777     else
8778     {
8779       // Internal branch
8780       int target=(link_addr[i].target-start)>>2;
8781       assert(target>=0&&target<slen);
8782       assert(instr_addr[target]);
8783       //#ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
8784       //set_jump_target_fillslot(link_addr[i].addr,instr_addr[target],link_addr[i].ext>>1);
8785       //#else
8786       set_jump_target(link_addr[i].addr, instr_addr[target]);
8787       //#endif
8788     }
8789   }
8790   // External Branch Targets (jump_in)
8791   if(copy+slen*4>(void *)shadow+sizeof(shadow)) copy=shadow;
8792   for(i=0;i<slen;i++)
8793   {
8794     if(bt[i]||i==0)
8795     {
8796       if(instr_addr[i]) // TODO - delay slots (=null)
8797       {
8798         u_int vaddr=start+i*4;
8799         u_int page=get_page(vaddr);
8800         u_int vpage=get_vpage(vaddr);
8801         literal_pool(256);
8802         {
8803           assem_debug("%p (%d) <- %8x\n",instr_addr[i],i,start+i*4);
8804           assem_debug("jump_in: %x\n",start+i*4);
8805           ll_add(jump_dirty+vpage,vaddr,out);
8806           void *entry_point = do_dirty_stub(i);
8807           ll_add_flags(jump_in+page,vaddr,state_rflags,entry_point);
8808           // If there was an existing entry in the hash table,
8809           // replace it with the new address.
8810           // Don't add new entries.  We'll insert the
8811           // ones that actually get used in check_addr().
8812           struct ht_entry *ht_bin = hash_table_get(vaddr);
8813           if (ht_bin->vaddr[0] == vaddr)
8814             ht_bin->tcaddr[0] = entry_point;
8815           if (ht_bin->vaddr[1] == vaddr)
8816             ht_bin->tcaddr[1] = entry_point;
8817         }
8818       }
8819     }
8820   }
8821   // Write out the literal pool if necessary
8822   literal_pool(0);
8823   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
8824   // Align code
8825   if(((u_int)out)&7) emit_addnop(13);
8826   #endif
8827   assert(out - (u_char *)beginning < MAX_OUTPUT_BLOCK_SIZE);
8828   //printf("shadow buffer: %p-%p\n",copy,(u_char *)copy+slen*4);
8829   memcpy(copy,source,slen*4);
8830   copy+=slen*4;
8831
8832   end_block(beginning);
8833
8834   // If we're within 256K of the end of the buffer,
8835   // start over from the beginning. (Is 256K enough?)
8836   if (out > translation_cache+(1<<TARGET_SIZE_2)-MAX_OUTPUT_BLOCK_SIZE)
8837     out = translation_cache;
8838
8839   // Trap writes to any of the pages we compiled
8840   for(i=start>>12;i<=(start+slen*4)>>12;i++) {
8841     invalid_code[i]=0;
8842   }
8843   inv_code_start=inv_code_end=~0;
8844
8845   // for PCSX we need to mark all mirrors too
8846   if(get_page(start)<(RAM_SIZE>>12))
8847     for(i=start>>12;i<=(start+slen*4)>>12;i++)
8848       invalid_code[((u_int)0x00000000>>12)|(i&0x1ff)]=
8849       invalid_code[((u_int)0x80000000>>12)|(i&0x1ff)]=
8850       invalid_code[((u_int)0xa0000000>>12)|(i&0x1ff)]=0;
8851
8852   /* Pass 10 - Free memory by expiring oldest blocks */
8853
8854   int end=(((out-translation_cache)>>(TARGET_SIZE_2-16))+16384)&65535;
8855   while(expirep!=end)
8856   {
8857     int shift=TARGET_SIZE_2-3; // Divide into 8 blocks
8858     uintptr_t base=(uintptr_t)translation_cache+((expirep>>13)<<shift); // Base address of this block
8859     inv_debug("EXP: Phase %d\n",expirep);
8860     switch((expirep>>11)&3)
8861     {
8862       case 0:
8863         // Clear jump_in and jump_dirty
8864         ll_remove_matching_addrs(jump_in+(expirep&2047),base,shift);
8865         ll_remove_matching_addrs(jump_dirty+(expirep&2047),base,shift);
8866         ll_remove_matching_addrs(jump_in+2048+(expirep&2047),base,shift);
8867         ll_remove_matching_addrs(jump_dirty+2048+(expirep&2047),base,shift);
8868         break;
8869       case 1:
8870         // Clear pointers
8871         ll_kill_pointers(jump_out[expirep&2047],base,shift);
8872         ll_kill_pointers(jump_out[(expirep&2047)+2048],base,shift);
8873         break;
8874       case 2:
8875         // Clear hash table
8876         for(i=0;i<32;i++) {
8877           struct ht_entry *ht_bin = &hash_table[((expirep&2047)<<5)+i];
8878           if (((uintptr_t)ht_bin->tcaddr[1]>>shift) == (base>>shift) ||
8879              (((uintptr_t)ht_bin->tcaddr[1]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
8880             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[1],ht_bin->tcaddr[1]);
8881             ht_bin->vaddr[1] = -1;
8882             ht_bin->tcaddr[1] = NULL;
8883           }
8884           if (((uintptr_t)ht_bin->tcaddr[0]>>shift) == (base>>shift) ||
8885              (((uintptr_t)ht_bin->tcaddr[0]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
8886             inv_debug("EXP: Remove hash %x -> %p\n",ht_bin->vaddr[0],ht_bin->tcaddr[0]);
8887             ht_bin->vaddr[0] = ht_bin->vaddr[1];
8888             ht_bin->tcaddr[0] = ht_bin->tcaddr[1];
8889             ht_bin->vaddr[1] = -1;
8890             ht_bin->tcaddr[1] = NULL;
8891           }
8892         }
8893         break;
8894       case 3:
8895         // Clear jump_out
8896         #ifdef __arm__
8897         if((expirep&2047)==0)
8898           do_clear_cache();
8899         #endif
8900         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
8901         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
8902         break;
8903     }
8904     expirep=(expirep+1)&65535;
8905   }
8906   return 0;
8907 }
8908
8909 // vim:shiftwidth=2:expandtab