source/mupen64plus-core/src/r4300/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2011 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdio.h>
  22 #include <string.h>
  23 #include <stdarg.h>
  24 #include <stdlib.h>
  25 #include <stdint.h> //include for uint64_t
  26 //#include <assert.h>
  27 #define assert(a)       {}
  28
  29 #include "../recomp.h"
  30 #include "../recomph.h" //include for function prototypes
  31 #include "../macros.h"
  32 #include "../r4300.h"
  33 #include "../ops.h"
  34 #include "../interupt.h"
  35 #include "new_dynarec.h"
  36
  37 #include "../../memory/memory.h"
  38 #include "../../main/rom.h"
  39
  40 #include <sys/mman.h>
  41
  42 #if NEW_DYNAREC == NEW_DYNAREC_X86
  43 #include "assem_x86.h"
  44 #elif NEW_DYNAREC == NEW_DYNAREC_ARM
  45 #include "assem_arm.h"
  46 #else
  47 #error Unsupported dynarec architecture
  48 #endif
  49
  50 #define MAXBLOCK 4096
  51 #define MAX_OUTPUT_BLOCK_SIZE 262144
  52 #define CLOCK_DIVIDER count_per_op
  53
  54 void *base_addr;
  55
  56 struct regstat
  57 {
  58   signed char regmap_entry[HOST_REGS];
  59   signed char regmap[HOST_REGS];
  60   uint64_t was32;
  61   uint64_t is32;
  62   uint64_t wasdirty;
  63   uint64_t dirty;
  64   uint64_t u;
  65   uint64_t uu;
  66   u_int wasconst;
  67   u_int isconst;
  68   uint64_t constmap[HOST_REGS];
  69 };
  70
  71 struct ll_entry
  72 {
  73   u_int vaddr;
  74   u_int reg32;
  75   void *addr;
  76   struct ll_entry *next;
  77 };
  78
  79 static u_int start;
  80 static u_int *source;
  81 static u_int pagelimit;
  82 static char insn[MAXBLOCK][10];
  83 static u_char itype[MAXBLOCK];
  84 static u_char opcode[MAXBLOCK];
  85 static u_char opcode2[MAXBLOCK];
  86 static u_char bt[MAXBLOCK];
  87 static u_char rs1[MAXBLOCK];
  88 static u_char rs2[MAXBLOCK];
  89 static u_char rt1[MAXBLOCK];
  90 static u_char rt2[MAXBLOCK];
  91 static u_char us1[MAXBLOCK];
  92 static u_char us2[MAXBLOCK];
  93 static u_char dep1[MAXBLOCK];
  94 static u_char dep2[MAXBLOCK];
  95 static u_char lt1[MAXBLOCK];
  96 static int imm[MAXBLOCK];
  97 static u_int ba[MAXBLOCK];
  98 static char likely[MAXBLOCK];
  99 static char is_ds[MAXBLOCK];
 100 static char ooo[MAXBLOCK];
 101 static uint64_t unneeded_reg[MAXBLOCK];
 102 static uint64_t unneeded_reg_upper[MAXBLOCK];
 103 static uint64_t branch_unneeded_reg[MAXBLOCK];
 104 static uint64_t branch_unneeded_reg_upper[MAXBLOCK];
 105 static uint64_t p32[MAXBLOCK];
 106 static uint64_t pr32[MAXBLOCK];
 107 static signed char regmap_pre[MAXBLOCK][HOST_REGS];
 108 #ifdef ASSEM_DEBUG
 109 static signed char regmap[MAXBLOCK][HOST_REGS];
 110 static signed char regmap_entry[MAXBLOCK][HOST_REGS];
 111 #endif
 112 static uint64_t constmap[MAXBLOCK][HOST_REGS];
 113 static struct regstat regs[MAXBLOCK];
 114 static struct regstat branch_regs[MAXBLOCK];
 115 static signed char minimum_free_regs[MAXBLOCK];
 116 static u_int needed_reg[MAXBLOCK];
 117 static uint64_t requires_32bit[MAXBLOCK];
 118 static u_int wont_dirty[MAXBLOCK];
 119 static u_int will_dirty[MAXBLOCK];
 120 static int ccadj[MAXBLOCK];
 121 static int slen;
 122 static u_int instr_addr[MAXBLOCK];
 123 static u_int link_addr[MAXBLOCK][3];
 124 static int linkcount;
 125 static u_int stubs[MAXBLOCK*3][8];
 126 static int stubcount;
 127 static int literalcount;
 128 static int is_delayslot;
 129 static int cop1_usable;
 130 u_char *out;
 131 struct ll_entry *jump_in[4096];
 132 static struct ll_entry *jump_out[4096];
 133 struct ll_entry *jump_dirty[4096];
 134 u_int hash_table[65536][4]  __attribute__((aligned(16)));
 135 static char shadow[2097152]  __attribute__((aligned(16)));
 136 static void *copy;
 137 static int expirep;
 138 u_int using_tlb;
 139 static u_int stop_after_jal;
 140 extern u_char restore_candidate[512];
 141 extern int cycle_count;
 142
 143   /* registers that may be allocated */
 144   /* 1-31 gpr */
 145 #define HIREG 32 // hi
 146 #define LOREG 33 // lo
 147 #define FSREG 34 // FPU status (FCSR)
 148 #define CSREG 35 // Coprocessor status
 149 #define CCREG 36 // Cycle count
 150 #define INVCP 37 // Pointer to invalid_code
 151 #define MMREG 38 // Pointer to memory_map
 152 #define ROREG 39 // ram offset (if rdram!=0x80000000)
 153 #define TEMPREG 40
 154 #define FTEMP 40 // FPU temporary register
 155 #define PTEMP 41 // Prefetch temporary register
 156 #define TLREG 42 // TLB mapping offset
 157 #define RHASH 43 // Return address hash
 158 #define RHTBL 44 // Return address hash table address
 159 #define RTEMP 45 // JR/JALR address register
 160 #define MAXREG 45
 161 #define AGEN1 46 // Address generation temporary register
 162 #define AGEN2 47 // Address generation temporary register
 163 #define MGEN1 48 // Maptable address generation temporary register
 164 #define MGEN2 49 // Maptable address generation temporary register
 165 #define BTREG 50 // Branch target temporary register
 166
 167   /* instruction types */
 168 #define NOP 0     // No operation
 169 #define LOAD 1    // Load
 170 #define STORE 2   // Store
 171 #define LOADLR 3  // Unaligned load
 172 #define STORELR 4 // Unaligned store
 173 #define MOV 5     // Move
 174 #define ALU 6     // Arithmetic/logic
 175 #define MULTDIV 7 // Multiply/divide
 176 #define SHIFT 8   // Shift by register
 177 #define SHIFTIMM 9// Shift by immediate
 178 #define IMM16 10  // 16-bit immediate
 179 #define RJUMP 11  // Unconditional jump to register
 180 #define UJUMP 12  // Unconditional jump
 181 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 182 #define SJUMP 14  // Conditional branch (regimm format)
 183 #define COP0 15   // Coprocessor 0
 184 #define COP1 16   // Coprocessor 1
 185 #define C1LS 17   // Coprocessor 1 load/store
 186 #define FJUMP 18  // Conditional branch (floating point)
 187 #define FLOAT 19  // Floating point unit
 188 #define FCONV 20  // Convert integer to float
 189 #define FCOMP 21  // Floating point compare (sets FSREG)
 190 #define SYSCALL 22// SYSCALL
 191 #define OTHER 23  // Other
 192 #define SPAN 24   // Branch/delay slot spans 2 pages
 193 #define NI 25     // Not implemented
 194
 195   /* stubs */
 196 #define CC_STUB 1
 197 #define FP_STUB 2
 198 #define LOADB_STUB 3
 199 #define LOADH_STUB 4
 200 #define LOADW_STUB 5
 201 #define LOADD_STUB 6
 202 #define LOADBU_STUB 7
 203 #define LOADHU_STUB 8
 204 #define STOREB_STUB 9
 205 #define STOREH_STUB 10
 206 #define STOREW_STUB 11
 207 #define STORED_STUB 12
 208 #define STORELR_STUB 13
 209 #define INVCODE_STUB 14
 210
 211   /* branch codes */
 212 #define TAKEN 1
 213 #define NOTTAKEN 2
 214 #define NULLDS 3
 215
 216 /* bug-fix to implement __clear_cache (missing in Android; http://code.google.com/p/android/issues/detail?id=1803) */
 217 void __clear_cache_bugfix(char* begin, char *end);
 218 #ifdef ANDROID
 219         #define __clear_cache __clear_cache_bugfix
 220 #endif
 221
 222 // asm linkage
 223 int new_recompile_block(int addr);
 224 void *get_addr_ht(u_int vaddr);
 225 static void remove_hash(int vaddr);
 226 void dyna_linker();
 227 void dyna_linker_ds();
 228 void verify_code();
 229 void verify_code_vm();
 230 void verify_code_ds();
 231 void cc_interrupt();
 232 void fp_exception();
 233 void fp_exception_ds();
 234 void jump_syscall();
 235 void jump_eret();
 236 #if NEW_DYNAREC == NEW_DYNAREC_ARM
 237 static void invalidate_addr(u_int addr);
 238 #endif
 239
 240 // TLB
 241 void TLBWI_new();
 242 void TLBWR_new();
 243 void read_nomem_new();
 244 void read_nomemb_new();
 245 void read_nomemh_new();
 246 void read_nomemd_new();
 247 void write_nomem_new();
 248 void write_nomemb_new();
 249 void write_nomemh_new();
 250 void write_nomemd_new();
 251 void write_rdram_new();
 252 void write_rdramb_new();
 253 void write_rdramh_new();
 254 void write_rdramd_new();
 255 extern u_int memory_map[1048576];
 256
 257 // Needed by assembler
 258 static void wb_register(signed char r,signed char regmap[],uint64_t dirty,uint64_t is32);
 259 static void wb_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty);
 260 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr);
 261 static void load_all_regs(signed char i_regmap[]);
 262 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 263 static void load_regs_entry(int t);
 264 static void load_all_consts(signed char regmap[],int is32,u_int dirty,int i);
 265
 266 static void add_stub(int type,int addr,int retaddr,int a,int b,int c,int d,int e);
 267 static void add_to_linker(int addr,int target,int ext);
 268 static int verify_dirty(void *addr);
 269
 270 //static int tracedebug=0;
 271
 272 //#define DEBUG_CYCLE_COUNT 1
 273
 274 // Uncomment these two lines to generate debug output:
 275 //#define ASSEM_DEBUG 1
 276 //#define INV_DEBUG 1
 277
 278 // Uncomment this line to output the number of NOTCOMPILED blocks as they occur:
 279 //#define COUNT_NOTCOMPILEDS 1
 280
 281 #if defined (COUNT_NOTCOMPILEDS )
 282         int notcompiledCount = 0;
 283 #endif
 284 static void nullf() {}
 285
 286 #if defined( ASSEM_DEBUG )
 287     #define assem_debug(...) DebugMessage(M64MSG_VERBOSE, __VA_ARGS__)
 288 #else
 289     #define assem_debug nullf
 290 #endif
 291 #if defined( INV_DEBUG )
 292     #define inv_debug(...) DebugMessage(M64MSG_VERBOSE, __VA_ARGS__)
 293 #else
 294     #define inv_debug nullf
 295 #endif
 296
 297 #define log_message(...) DebugMessage(M64MSG_VERBOSE, __VA_ARGS__)
 298
 299 static void tlb_hacks()
 300 {
 301   // Goldeneye hack
 302   if (strncmp((char *) ROM_HEADER.Name, "GOLDENEYE",9) == 0)
 303   {
 304     u_int addr;
 305     int n;
 306     switch (ROM_HEADER.Country_code&0xFF)
 307     {
 308       case 0x45: // U
 309         addr=0x34b30;
 310         break;
 311       case 0x4A: // J
 312         addr=0x34b70;
 313         break;
 314       case 0x50: // E
 315         addr=0x329f0;
 316         break;
 317       default:
 318         // Unknown country code
 319         addr=0;
 320         break;
 321     }
 322     u_int rom_addr=(u_int)rom;
 323     #ifdef ROM_COPY
 324     // Since memory_map is 32-bit, on 64-bit systems the rom needs to be
 325     // in the lower 4G of memory to use this hack.  Copy it if necessary.
 326     if((void *)rom>(void *)0xffffffff) {
 327       munmap(ROM_COPY, 67108864);
 328       if(mmap(ROM_COPY, 12582912,
 329               PROT_READ | PROT_WRITE,
 330               MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
 331               -1, 0) <= 0) {DebugMessage(M64MSG_ERROR, "mmap() failed");}
 332       memcpy(ROM_COPY,rom,12582912);
 333       rom_addr=(u_int)ROM_COPY;
 334     }
 335     #endif
 336     if(addr) {
 337       for(n=0x7F000;n<0x80000;n++) {
 338         memory_map[n]=(((u_int)(rom_addr+addr-0x7F000000))>>2)|0x40000000;
 339       }
 340     }
 341   }
 342 }
 343
 344 // Get address from virtual address
 345 // This is called from the recompiled JR/JALR instructions
 346 void *get_addr(u_int vaddr)
 347 {
 348   u_int page=(vaddr^0x80000000)>>12;
 349   u_int vpage=page;
 350   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[vaddr>>12]^0x80000000)>>12;
 351   if(page>2048) page=2048+(page&2047);
 352   if(vpage>262143&&tlb_LUT_r[vaddr>>12]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
 353   if(vpage>2048) vpage=2048+(vpage&2047);
 354   struct ll_entry *head;
 355   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr %x,page %d)",Count,next_interupt,vaddr,page);
 356   head=jump_in[page];
 357   while(head!=NULL) {
 358     if(head->vaddr==vaddr&&head->reg32==0) {
 359   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr match %x: %x)",Count,next_interupt,vaddr,(int)head->addr);
 360       u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 361       ht_bin[3]=ht_bin[1];
 362       ht_bin[2]=ht_bin[0];
 363       ht_bin[1]=(int)head->addr;
 364       ht_bin[0]=vaddr;
 365       return head->addr;
 366     }
 367     head=head->next;
 368   }
 369   head=jump_dirty[vpage];
 370   while(head!=NULL) {
 371     if(head->vaddr==vaddr&&head->reg32==0) {
 372       //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr match dirty %x: %x)",Count,next_interupt,vaddr,(int)head->addr);
 373       // Don't restore blocks which are about to expire from the cache
 374       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 375       if(verify_dirty(head->addr)) {
 376         //DebugMessage(M64MSG_VERBOSE, "restore candidate: %x (%d) d=%d",vaddr,page,invalid_code[vaddr>>12]);
 377         invalid_code[vaddr>>12]=0;
 378         memory_map[vaddr>>12]|=0x40000000;
 379         if(vpage<2048) {
 380           if(tlb_LUT_r[vaddr>>12]) {
 381             invalid_code[tlb_LUT_r[vaddr>>12]>>12]=0;
 382             memory_map[tlb_LUT_r[vaddr>>12]>>12]|=0x40000000;
 383           }
 384           restore_candidate[vpage>>3]|=1<<(vpage&7);
 385         }
 386         else restore_candidate[page>>3]|=1<<(page&7);
 387         u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 388         if(ht_bin[0]==vaddr) {
 389           ht_bin[1]=(int)head->addr; // Replace existing entry
 390         }
 391         else
 392         {
 393           ht_bin[3]=ht_bin[1];
 394           ht_bin[2]=ht_bin[0];
 395           ht_bin[1]=(int)head->addr;
 396           ht_bin[0]=vaddr;
 397         }
 398         return head->addr;
 399       }
 400     }
 401     head=head->next;
 402   }
 403   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr no-match %x)",Count,next_interupt,vaddr);
 404   int r=new_recompile_block(vaddr);
 405   if(r==0) return get_addr(vaddr);
 406   // Execute in unmapped page, generate pagefault execption
 407   Status|=2;
 408   Cause=(vaddr<<31)|0x8;
 409   EPC=(vaddr&1)?vaddr-5:vaddr;
 410   BadVAddr=(vaddr&~1);
 411   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 412   EntryHi=BadVAddr&0xFFFFE000;
 413   return get_addr_ht(0x80000000);
 414 }
 415 // Look up address in hash table first
 416 void *get_addr_ht(u_int vaddr)
 417 {
 418   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr_ht %x)",Count,next_interupt,vaddr);
 419   u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 420   if(ht_bin[0]==vaddr) return (void *)ht_bin[1];
 421   if(ht_bin[2]==vaddr) return (void *)ht_bin[3];
 422   return get_addr(vaddr);
 423 }
 424
 425 void *get_addr_32(u_int vaddr,u_int flags)
 426 {
 427   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr_32 %x,flags %x)",Count,next_interupt,vaddr,flags);
 428   u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 429   if(ht_bin[0]==vaddr) return (void *)ht_bin[1];
 430   if(ht_bin[2]==vaddr) return (void *)ht_bin[3];
 431   u_int page=(vaddr^0x80000000)>>12;
 432   u_int vpage=page;
 433   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[vaddr>>12]^0x80000000)>>12;
 434   if(page>2048) page=2048+(page&2047);
 435   if(vpage>262143&&tlb_LUT_r[vaddr>>12]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
 436   if(vpage>2048) vpage=2048+(vpage&2047);
 437   struct ll_entry *head;
 438   head=jump_in[page];
 439   while(head!=NULL) {
 440     if(head->vaddr==vaddr&&(head->reg32&flags)==0) {
 441       //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr_32 match %x: %x)",Count,next_interupt,vaddr,(int)head->addr);
 442       if(head->reg32==0) {
 443         u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 444         if(ht_bin[0]==-1) {
 445           ht_bin[1]=(int)head->addr;
 446           ht_bin[0]=vaddr;
 447         }else if(ht_bin[2]==-1) {
 448           ht_bin[3]=(int)head->addr;
 449           ht_bin[2]=vaddr;
 450         }
 451         //ht_bin[3]=ht_bin[1];
 452         //ht_bin[2]=ht_bin[0];
 453         //ht_bin[1]=(int)head->addr;
 454         //ht_bin[0]=vaddr;
 455       }
 456       return head->addr;
 457     }
 458     head=head->next;
 459   }
 460   head=jump_dirty[vpage];
 461   while(head!=NULL) {
 462     if(head->vaddr==vaddr&&(head->reg32&flags)==0) {
 463       //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr_32 match dirty %x: %x)",Count,next_interupt,vaddr,(int)head->addr);
 464       // Don't restore blocks which are about to expire from the cache
 465       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 466       if(verify_dirty(head->addr)) {
 467         //DebugMessage(M64MSG_VERBOSE, "restore candidate: %x (%d) d=%d",vaddr,page,invalid_code[vaddr>>12]);
 468         invalid_code[vaddr>>12]=0;
 469         memory_map[vaddr>>12]|=0x40000000;
 470         if(vpage<2048) {
 471           if(tlb_LUT_r[vaddr>>12]) {
 472             invalid_code[tlb_LUT_r[vaddr>>12]>>12]=0;
 473             memory_map[tlb_LUT_r[vaddr>>12]>>12]|=0x40000000;
 474           }
 475           restore_candidate[vpage>>3]|=1<<(vpage&7);
 476         }
 477         else restore_candidate[page>>3]|=1<<(page&7);
 478         if(head->reg32==0) {
 479           u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 480           if(ht_bin[0]==-1) {
 481             ht_bin[1]=(int)head->addr;
 482             ht_bin[0]=vaddr;
 483           }else if(ht_bin[2]==-1) {
 484             ht_bin[3]=(int)head->addr;
 485             ht_bin[2]=vaddr;
 486           }
 487           //ht_bin[3]=ht_bin[1];
 488           //ht_bin[2]=ht_bin[0];
 489           //ht_bin[1]=(int)head->addr;
 490           //ht_bin[0]=vaddr;
 491         }
 492         return head->addr;
 493       }
 494     }
 495     head=head->next;
 496   }
 497   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (get_addr_32 no-match %x,flags %x)",Count,next_interupt,vaddr,flags);
 498   int r=new_recompile_block(vaddr);
 499   if(r==0) return get_addr(vaddr);
 500   // Execute in unmapped page, generate pagefault execption
 501   Status|=2;
 502   Cause=(vaddr<<31)|0x8;
 503   EPC=(vaddr&1)?vaddr-5:vaddr;
 504   BadVAddr=(vaddr&~1);
 505   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 506   EntryHi=BadVAddr&0xFFFFE000;
 507   return get_addr_ht(0x80000000);
 508 }
 509
 510 static void clear_all_regs(signed char regmap[])
 511 {
 512   int hr;
 513   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 514 }
 515
 516 static signed char get_reg(signed char regmap[],int r)
 517 {
 518   int hr;
 519   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 520   return -1;
 521 }
 522
 523 // Find a register that is available for two consecutive cycles
 524 static signed char get_reg2(signed char regmap1[],signed char regmap2[],int r)
 525 {
 526   int hr;
 527   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 528   return -1;
 529 }
 530
 531 static int count_free_regs(signed char regmap[])
 532 {
 533   int count=0;
 534   int hr;
 535   for(hr=0;hr<HOST_REGS;hr++)
 536   {
 537     if(hr!=EXCLUDE_REG) {
 538       if(regmap[hr]<0) count++;
 539     }
 540   }
 541   return count;
 542 }
 543
 544 static void dirty_reg(struct regstat *cur,signed char reg)
 545 {
 546   int hr;
 547   if(!reg) return;
 548   for (hr=0;hr<HOST_REGS;hr++) {
 549     if((cur->regmap[hr]&63)==reg) {
 550       cur->dirty|=1<<hr;
 551     }
 552   }
 553 }
 554
 555 // If we dirty the lower half of a 64 bit register which is now being
 556 // sign-extended, we need to dump the upper half.
 557 // Note: Do this only after completion of the instruction, because
 558 // some instructions may need to read the full 64-bit value even if
 559 // overwriting it (eg SLTI, DSRA32).
 560 static void flush_dirty_uppers(struct regstat *cur)
 561 {
 562   int hr,reg;
 563   for (hr=0;hr<HOST_REGS;hr++) {
 564     if((cur->dirty>>hr)&1) {
 565       reg=cur->regmap[hr];
 566       if(reg>=64)
 567         if((cur->is32>>(reg&63))&1) cur->regmap[hr]=-1;
 568     }
 569   }
 570 }
 571
 572 static void set_const(struct regstat *cur,signed char reg,uint64_t value)
 573 {
 574   int hr;
 575   if(!reg) return;
 576   for (hr=0;hr<HOST_REGS;hr++) {
 577     if(cur->regmap[hr]==reg) {
 578       cur->isconst|=1<<hr;
 579       cur->constmap[hr]=value;
 580     }
 581     else if((cur->regmap[hr]^64)==reg) {
 582       cur->isconst|=1<<hr;
 583       cur->constmap[hr]=value>>32;
 584     }
 585   }
 586 }
 587
 588 static void clear_const(struct regstat *cur,signed char reg)
 589 {
 590   int hr;
 591   if(!reg) return;
 592   for (hr=0;hr<HOST_REGS;hr++) {
 593     if((cur->regmap[hr]&63)==reg) {
 594       cur->isconst&=~(1<<hr);
 595     }
 596   }
 597 }
 598
 599 static int is_const(struct regstat *cur,signed char reg)
 600 {
 601   int hr;
 602   if(reg<0) return 0;
 603   if(!reg) return 1;
 604   for (hr=0;hr<HOST_REGS;hr++) {
 605     if((cur->regmap[hr]&63)==reg) {
 606       return (cur->isconst>>hr)&1;
 607     }
 608   }
 609   return 0;
 610 }
 611 static uint64_t get_const(struct regstat *cur,signed char reg)
 612 {
 613   int hr;
 614   if(!reg) return 0;
 615   for (hr=0;hr<HOST_REGS;hr++) {
 616     if(cur->regmap[hr]==reg) {
 617       return cur->constmap[hr];
 618     }
 619   }
 620   DebugMessage(M64MSG_ERROR, "Unknown constant in r%d",reg);
 621   exit(1);
 622 }
 623
 624 // Least soon needed registers
 625 // Look at the next ten instructions and see which registers
 626 // will be used.  Try not to reallocate these.
 627 static void lsn(u_char hsn[], int i, int *preferred_reg)
 628 {
 629   int j;
 630   int b=-1;
 631   for(j=0;j<9;j++)
 632   {
 633     if(i+j>=slen) {
 634       j=slen-i-1;
 635       break;
 636     }
 637     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 638     {
 639       // Don't go past an unconditonal jump
 640       j++;
 641       break;
 642     }
 643   }
 644   for(;j>=0;j--)
 645   {
 646     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 647     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 648     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 649     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 650     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 651       // Stores can allocate zero
 652       hsn[rs1[i+j]]=j;
 653       hsn[rs2[i+j]]=j;
 654     }
 655     // On some architectures stores need invc_ptr
 656     #if defined(HOST_IMM8)
 657     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39) {
 658       hsn[INVCP]=j;
 659     }
 660     #endif
 661     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP||itype[i+j]==FJUMP))
 662     {
 663       hsn[CCREG]=j;
 664       b=j;
 665     }
 666   }
 667   if(b>=0)
 668   {
 669     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 670     {
 671       // Follow first branch
 672       int t=(ba[i+b]-start)>>2;
 673       j=7-b;if(t+j>=slen) j=slen-t-1;
 674       for(;j>=0;j--)
 675       {
 676         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 677         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 678         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 679         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 680       }
 681     }
 682     // TODO: preferred register based on backward branch
 683   }
 684   // Delay slot should preferably not overwrite branch conditions or cycle count
 685   if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP)) {
 686     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 687     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 688     hsn[CCREG]=1;
 689     // ...or hash tables
 690     hsn[RHASH]=1;
 691     hsn[RHTBL]=1;
 692   }
 693   // Coprocessor load/store needs FTEMP, even if not declared
 694   if(itype[i]==C1LS) {
 695     hsn[FTEMP]=0;
 696   }
 697   // Load L/R also uses FTEMP as a temporary register
 698   if(itype[i]==LOADLR) {
 699     hsn[FTEMP]=0;
 700   }
 701   // Also 64-bit SDL/SDR
 702   if(opcode[i]==0x2c||opcode[i]==0x2d) {
 703     hsn[FTEMP]=0;
 704   }
 705   // Don't remove the TLB registers either
 706   if(itype[i]==LOAD || itype[i]==LOADLR || itype[i]==STORE || itype[i]==STORELR || itype[i]==C1LS ) {
 707     hsn[TLREG]=0;
 708   }
 709   // Don't remove the miniht registers
 710   if(itype[i]==UJUMP||itype[i]==RJUMP)
 711   {
 712     hsn[RHASH]=0;
 713     hsn[RHTBL]=0;
 714   }
 715 }
 716
 717 // We only want to allocate registers if we're going to use them again soon
 718 static int needed_again(int r, int i)
 719 {
 720   int j;
 721   /*int b=-1;*/
 722   int rn=10;
 723
 724   if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000))
 725   {
 726     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 727       return 0; // Don't need any registers if exiting the block
 728   }
 729   for(j=0;j<9;j++)
 730   {
 731     if(i+j>=slen) {
 732       j=slen-i-1;
 733       break;
 734     }
 735     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 736     {
 737       // Don't go past an unconditonal jump
 738       j++;
 739       break;
 740     }
 741     if(itype[i+j]==SYSCALL||((source[i+j]&0xfc00003f)==0x0d))
 742     {
 743       break;
 744     }
 745   }
 746   for(;j>=1;j--)
 747   {
 748     if(rs1[i+j]==r) rn=j;
 749     if(rs2[i+j]==r) rn=j;
 750     if((unneeded_reg[i+j]>>r)&1) rn=10;
 751     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP||itype[i+j]==FJUMP))
 752     {
 753       /*b=j;*/
 754     }
 755   }
 756   /*
 757   if(b>=0)
 758   {
 759     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 760     {
 761       // Follow first branch
 762       int o=rn;
 763       int t=(ba[i+b]-start)>>2;
 764       j=7-b;if(t+j>=slen) j=slen-t-1;
 765       for(;j>=0;j--)
 766       {
 767         if(!((unneeded_reg[t+j]>>r)&1)) {
 768           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 769           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 770         }
 771         else rn=o;
 772       }
 773     }
 774   }*/
 775   if(rn<10) return 1;
 776   return 0;
 777 }
 778
 779 // Try to match register allocations at the end of a loop with those
 780 // at the beginning
 781 static int loop_reg(int i, int r, int hr)
 782 {
 783   int j,k;
 784   for(j=0;j<9;j++)
 785   {
 786     if(i+j>=slen) {
 787       j=slen-i-1;
 788       break;
 789     }
 790     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 791     {
 792       // Don't go past an unconditonal jump
 793       j++;
 794       break;
 795     }
 796   }
 797   k=0;
 798   if(i>0){
 799     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP)
 800       k--;
 801   }
 802   for(;k<j;k++)
 803   {
 804     if(r<64&&((unneeded_reg[i+k]>>r)&1)) return hr;
 805     if(r>64&&((unneeded_reg_upper[i+k]>>r)&1)) return hr;
 806     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP||itype[i+k]==FJUMP))
 807     {
 808       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 809       {
 810         int t=(ba[i+k]-start)>>2;
 811         int reg=get_reg(regs[t].regmap_entry,r);
 812         if(reg>=0) return reg;
 813         //reg=get_reg(regs[t+1].regmap_entry,r);
 814         //if(reg>=0) return reg;
 815       }
 816     }
 817   }
 818   return hr;
 819 }
 820
 821
 822 // Allocate every register, preserving source/target regs
 823 static void alloc_all(struct regstat *cur,int i)
 824 {
 825   int hr;
 826
 827   for(hr=0;hr<HOST_REGS;hr++) {
 828     if(hr!=EXCLUDE_REG) {
 829       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 830          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 831       {
 832         cur->regmap[hr]=-1;
 833         cur->dirty&=~(1<<hr);
 834       }
 835       // Don't need zeros
 836       if((cur->regmap[hr]&63)==0)
 837       {
 838         cur->regmap[hr]=-1;
 839         cur->dirty&=~(1<<hr);
 840       }
 841     }
 842   }
 843 }
 844
 845
 846 static void div64(int64_t dividend,int64_t divisor)
 847 {
 848   if ((dividend) && (divisor)) {
 849   lo=dividend/divisor;
 850   hi=dividend%divisor;
 851   } else {
 852   lo=0;
 853   hi=0;
 854   }
 855   //DebugMessage(M64MSG_VERBOSE, "TRACE: ddiv %8x%8x %8x%8x" ,(int)reg[HIREG],(int)(reg[HIREG]>>32)
 856   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 857 }
 858 static void divu64(uint64_t dividend,uint64_t divisor)
 859 {
 860   if ((dividend) && (divisor)) {
 861   lo=dividend/divisor;
 862   hi=dividend%divisor;
 863   } else {
 864   lo=0;
 865   hi=0;
 866   }
 867   //DebugMessage(M64MSG_VERBOSE, "TRACE: ddivu %8x%8x %8x%8x",(int)reg[HIREG],(int)(reg[HIREG]>>32)
 868   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 869 }
 870 static void div32(int32_t dividend,int32_t divisor)
 871 {
 872   if ((dividend) && (divisor)) {
 873   lo=dividend/divisor;
 874   hi=dividend%divisor;
 875   } else {
 876   lo=0;
 877   hi=0;
 878   }
 879   //DebugMessage(M64MSG_VERBOSE, "TRACE: ddiv %8x%8x %8x%8x" ,(int)reg[HIREG],(int)(reg[HIREG]>>32)
 880   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 881 }
 882 static void divu32(uint32_t dividend,uint32_t divisor)
 883 {
 884   if ((dividend) && (divisor)) {
 885   lo=dividend/divisor;
 886   hi=dividend%divisor;
 887   } else {
 888   lo=0;
 889   hi=0;
 890   }
 891   //DebugMessage(M64MSG_VERBOSE, "TRACE: ddivu %8x%8x %8x%8x",(int)reg[HIREG],(int)(reg[HIREG]>>32)
 892   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 893 }
 894
 895 static void mult64(int64_t m1,int64_t m2)
 896 {
 897    uint64_t op1, op2, op3, op4;
 898    uint64_t result1, result2, result3, result4;
 899    uint64_t temp1, temp2, temp3, temp4;
 900    int sign = 0;
 901
 902    if (m1 < 0)
 903      {
 904     op2 = -m1;
 905     sign = 1 - sign;
 906      }
 907    else op2 = m1;
 908    if (m2 < 0)
 909      {
 910     op4 = -m2;
 911     sign = 1 - sign;
 912      }
 913    else op4 = m2;
 914
 915    op1 = op2 & 0xFFFFFFFF;
 916    op2 = (op2 >> 32) & 0xFFFFFFFF;
 917    op3 = op4 & 0xFFFFFFFF;
 918    op4 = (op4 >> 32) & 0xFFFFFFFF;
 919
 920    temp1 = op1 * op3;
 921    temp2 = (temp1 >> 32) + op1 * op4;
 922    temp3 = op2 * op3;
 923    temp4 = (temp3 >> 32) + op2 * op4;
 924
 925    result1 = temp1 & 0xFFFFFFFF;
 926    result2 = temp2 + (temp3 & 0xFFFFFFFF);
 927    result3 = (result2 >> 32) + temp4;
 928    result4 = (result3 >> 32);
 929
 930    lo = result1 | (result2 << 32);
 931    hi = (result3 & 0xFFFFFFFF) | (result4 << 32);
 932    if (sign)
 933      {
 934     hi = ~hi;
 935     if (!lo) hi++;
 936     else lo = ~lo + 1;
 937      }
 938 }
 939
 940 #if NEW_DYNAREC == NEW_DYNAREC_ARM
 941 static void multu64(uint64_t m1,uint64_t m2)
 942 {
 943    uint64_t op1, op2, op3, op4;
 944    uint64_t result1, result2, result3, result4;
 945    uint64_t temp1, temp2, temp3, temp4;
 946
 947    op1 = m1 & 0xFFFFFFFF;
 948    op2 = (m1 >> 32) & 0xFFFFFFFF;
 949    op3 = m2 & 0xFFFFFFFF;
 950    op4 = (m2 >> 32) & 0xFFFFFFFF;
 951
 952    temp1 = op1 * op3;
 953    temp2 = (temp1 >> 32) + op1 * op4;
 954    temp3 = op2 * op3;
 955    temp4 = (temp3 >> 32) + op2 * op4;
 956
 957    result1 = temp1 & 0xFFFFFFFF;
 958    result2 = temp2 + (temp3 & 0xFFFFFFFF);
 959    result3 = (result2 >> 32) + temp4;
 960    result4 = (result3 >> 32);
 961
 962    lo = result1 | (result2 << 32);
 963    hi = (result3 & 0xFFFFFFFF) | (result4 << 32);
 964
 965   //DebugMessage(M64MSG_VERBOSE, "TRACE: dmultu %8x%8x %8x%8x",(int)reg[HIREG],(int)(reg[HIREG]>>32)
 966   //                                      ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 967 }
 968 #endif
 969
 970 static uint64_t ldl_merge(uint64_t original,uint64_t loaded,u_int bits)
 971 {
 972   if(bits) {
 973     original<<=64-bits;
 974     original>>=64-bits;
 975     loaded<<=bits;
 976     original|=loaded;
 977   }
 978   else original=loaded;
 979   return original;
 980 }
 981 static uint64_t ldr_merge(uint64_t original,uint64_t loaded,u_int bits)
 982 {
 983   if(bits^56) {
 984     original>>=64-(bits^56);
 985     original<<=64-(bits^56);
 986     loaded>>=bits^56;
 987     original|=loaded;
 988   }
 989   else original=loaded;
 990   return original;
 991 }
 992
 993 #if NEW_DYNAREC == NEW_DYNAREC_X86
 994 #include "assem_x86.c"
 995 #elif NEW_DYNAREC == NEW_DYNAREC_ARM
 996 #include "assem_arm.c"
 997 #else
 998 #error Unsupported dynarec architecture
 999 #endif
1000
1001 // Add virtual address mapping to linked list
1002 static void ll_add(struct ll_entry **head,int vaddr,void *addr)
1003 {
1004   struct ll_entry *new_entry;
1005   new_entry=malloc(sizeof(struct ll_entry));
1006   assert(new_entry!=NULL);
1007   new_entry->vaddr=vaddr;
1008   new_entry->reg32=0;
1009   new_entry->addr=addr;
1010   new_entry->next=*head;
1011   *head=new_entry;
1012 }
1013
1014 // Add virtual address mapping for 32-bit compiled block
1015 static void ll_add_32(struct ll_entry **head,int vaddr,u_int reg32,void *addr)
1016 {
1017   struct ll_entry *new_entry;
1018   new_entry=malloc(sizeof(struct ll_entry));
1019   assert(new_entry!=NULL);
1020   new_entry->vaddr=vaddr;
1021   new_entry->reg32=reg32;
1022   new_entry->addr=addr;
1023   new_entry->next=*head;
1024   *head=new_entry;
1025 }
1026
1027 // Check if an address is already compiled
1028 // but don't return addresses which are about to expire from the cache
1029 static void *check_addr(u_int vaddr)
1030 {
1031   u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
1032   if(ht_bin[0]==vaddr) {
1033     if(((ht_bin[1]-MAX_OUTPUT_BLOCK_SIZE-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
1034       if(isclean(ht_bin[1])) return (void *)ht_bin[1];
1035   }
1036   if(ht_bin[2]==vaddr) {
1037     if(((ht_bin[3]-MAX_OUTPUT_BLOCK_SIZE-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
1038       if(isclean(ht_bin[3])) return (void *)ht_bin[3];
1039   }
1040   u_int page=(vaddr^0x80000000)>>12;
1041   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[vaddr>>12]^0x80000000)>>12;
1042   if(page>2048) page=2048+(page&2047);
1043   struct ll_entry *head;
1044   head=jump_in[page];
1045   while(head!=NULL) {
1046     if(head->vaddr==vaddr&&head->reg32==0) {
1047       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1048         // Update existing entry with current address
1049         if(ht_bin[0]==vaddr) {
1050           ht_bin[1]=(int)head->addr;
1051           return head->addr;
1052         }
1053         if(ht_bin[2]==vaddr) {
1054           ht_bin[3]=(int)head->addr;
1055           return head->addr;
1056         }
1057         // Insert into hash table with low priority.
1058         // Don't evict existing entries, as they are probably
1059         // addresses that are being accessed frequently.
1060         if(ht_bin[0]==-1) {
1061           ht_bin[1]=(int)head->addr;
1062           ht_bin[0]=vaddr;
1063         }else if(ht_bin[2]==-1) {
1064           ht_bin[3]=(int)head->addr;
1065           ht_bin[2]=vaddr;
1066         }
1067         return head->addr;
1068       }
1069     }
1070     head=head->next;
1071   }
1072   return 0;
1073 }
1074
1075 static void remove_hash(int vaddr)
1076 {
1077   //DebugMessage(M64MSG_VERBOSE, "remove hash: %x",vaddr);
1078   u_int *ht_bin=hash_table[(((vaddr)>>16)^vaddr)&0xFFFF];
1079   if(ht_bin[2]==vaddr) {
1080     ht_bin[2]=ht_bin[3]=-1;
1081   }
1082   if(ht_bin[0]==vaddr) {
1083     ht_bin[0]=ht_bin[2];
1084     ht_bin[1]=ht_bin[3];
1085     ht_bin[2]=ht_bin[3]=-1;
1086   }
1087 }
1088
1089 static void ll_remove_matching_addrs(struct ll_entry **head,int addr,int shift)
1090 {
1091   struct ll_entry *next;
1092   while(*head) {
1093     if(((u_int)((*head)->addr)>>shift)==(addr>>shift) ||
1094        ((u_int)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
1095     {
1096       inv_debug("EXP: Remove pointer to %x (%x)\n",(int)(*head)->addr,(*head)->vaddr);
1097       remove_hash((*head)->vaddr);
1098       next=(*head)->next;
1099       free(*head);
1100       *head=next;
1101     }
1102     else
1103     {
1104       head=&((*head)->next);
1105     }
1106   }
1107 }
1108
1109 // Remove all entries from linked list
1110 static void ll_clear(struct ll_entry **head)
1111 {
1112   struct ll_entry *cur;
1113   struct ll_entry *next;
1114   if((cur=*head)) {
1115     *head=0;
1116     while(cur) {
1117       next=cur->next;
1118       free(cur);
1119       cur=next;
1120     }
1121   }
1122 }
1123
1124 // Dereference the pointers and remove if it matches
1125 static void ll_kill_pointers(struct ll_entry *head,int addr,int shift)
1126 {
1127   while(head) {
1128     int ptr=get_pointer(head->addr);
1129     inv_debug("EXP: Lookup pointer to %x at %x (%x)\n",(int)ptr,(int)head->addr,head->vaddr);
1130     if(((ptr>>shift)==(addr>>shift)) ||
1131        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
1132     {
1133       inv_debug("EXP: Kill pointer at %x (%x)\n",(int)head->addr,head->vaddr);
1134       u_int host_addr=(int)kill_pointer(head->addr);
1135       #if NEW_DYNAREC == NEW_DYNAREC_ARM
1136         needs_clear_cache[(host_addr-(u_int)base_addr)>>17]|=1<<(((host_addr-(u_int)base_addr)>>12)&31);
1137       #endif
1138     }
1139     head=head->next;
1140   }
1141 }
1142
1143 // This is called when we write to a compiled block (see do_invstub)
1144 static void invalidate_page(u_int page)
1145 {
1146   struct ll_entry *head;
1147   struct ll_entry *next;
1148   head=jump_in[page];
1149   jump_in[page]=0;
1150   while(head!=NULL) {
1151     inv_debug("INVALIDATE: %x\n",head->vaddr);
1152     remove_hash(head->vaddr);
1153     next=head->next;
1154     free(head);
1155     head=next;
1156   }
1157   head=jump_out[page];
1158   jump_out[page]=0;
1159   while(head!=NULL) {
1160     inv_debug("INVALIDATE: kill pointer to %x (%x)\n",head->vaddr,(int)head->addr);
1161     u_int host_addr=(int)kill_pointer(head->addr);
1162     #if NEW_DYNAREC == NEW_DYNAREC_ARM
1163       needs_clear_cache[(host_addr-(u_int)base_addr)>>17]|=1<<(((host_addr-(u_int)base_addr)>>12)&31);
1164     #endif
1165     next=head->next;
1166     free(head);
1167     head=next;
1168   }
1169 }
1170 void invalidate_block(u_int block)
1171 {
1172   u_int page,vpage;
1173   page=vpage=block^0x80000;
1174   if(page>262143&&tlb_LUT_r[block]) page=(tlb_LUT_r[block]^0x80000000)>>12;
1175   if(page>2048) page=2048+(page&2047);
1176   if(vpage>262143&&tlb_LUT_r[block]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
1177   if(vpage>2048) vpage=2048+(vpage&2047);
1178   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1179   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1180   u_int first,last;
1181   first=last=page;
1182   struct ll_entry *head;
1183   head=jump_dirty[vpage];
1184   //DebugMessage(M64MSG_VERBOSE, "page=%d vpage=%d",page,vpage);
1185   while(head!=NULL) {
1186     u_int start,end;
1187     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1188       get_bounds((int)head->addr,&start,&end);
1189       //DebugMessage(M64MSG_VERBOSE, "start: %x end: %x",start,end);
1190       if(page<2048&&start>=0x80000000&&end<0x80800000) {
1191         if(((start-(u_int)rdram)>>12)<=page&&((end-1-(u_int)rdram)>>12)>=page) {
1192           if((((start-(u_int)rdram)>>12)&2047)<first) first=((start-(u_int)rdram)>>12)&2047;
1193           if((((end-1-(u_int)rdram)>>12)&2047)>last) last=((end-1-(u_int)rdram)>>12)&2047;
1194         }
1195       }
1196       if(page<2048&&(signed int)start>=(signed int)0xC0000000&&(signed int)end>=(signed int)0xC0000000) {
1197         if(((start+memory_map[start>>12]-(u_int)rdram)>>12)<=page&&((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)>=page) {
1198           if((((start+memory_map[start>>12]-(u_int)rdram)>>12)&2047)<first) first=((start+memory_map[start>>12]-(u_int)rdram)>>12)&2047;
1199           if((((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)&2047)>last) last=((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)&2047;
1200         }
1201       }
1202     }
1203     head=head->next;
1204   }
1205   //DebugMessage(M64MSG_VERBOSE, "first=%d last=%d",first,last);
1206   invalidate_page(page);
1207   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1208   assert(last<page+5);
1209   // Invalidate the adjacent pages if a block crosses a 4K boundary
1210   while(first<page) {
1211     invalidate_page(first);
1212     first++;
1213   }
1214   for(first=page+1;first<last;first++) {
1215     invalidate_page(first);
1216   }
1217   #if NEW_DYNAREC == NEW_DYNAREC_ARM
1218     do_clear_cache();
1219   #endif
1220
1221   // Don't trap writes
1222   invalid_code[block]=1;
1223   // If there is a valid TLB entry for this page, remove write protect
1224   if(tlb_LUT_w[block]) {
1225     assert(tlb_LUT_r[block]==tlb_LUT_w[block]);
1226     // CHECK: Is this right?
1227     memory_map[block]=((tlb_LUT_w[block]&0xFFFFF000)-(block<<12)+(unsigned int)rdram-0x80000000)>>2;
1228     u_int real_block=tlb_LUT_w[block]>>12;
1229     invalid_code[real_block]=1;
1230     if(real_block>=0x80000&&real_block<0x80800) memory_map[real_block]=((u_int)rdram-0x80000000)>>2;
1231   }
1232   else if(block>=0x80000&&block<0x80800) memory_map[block]=((u_int)rdram-0x80000000)>>2;
1233   #ifdef USE_MINI_HT
1234   memset(mini_ht,-1,sizeof(mini_ht));
1235   #endif
1236 }
1237
1238 #if NEW_DYNAREC == NEW_DYNAREC_ARM
1239 static void invalidate_addr(u_int addr)
1240 {
1241   invalidate_block(addr>>12);
1242 }
1243 #endif
1244
1245 // This is called when loading a save state.
1246 // Anything could have changed, so invalidate everything.
1247 void invalidate_all_pages()
1248 {
1249   u_int page;
1250   for(page=0;page<4096;page++)
1251     invalidate_page(page);
1252   for(page=0;page<1048576;page++)
1253     if(!invalid_code[page]) {
1254       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1255       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1256     }
1257   #if NEW_DYNAREC == NEW_DYNAREC_ARM
1258   __clear_cache((void *)base_addr,(void *)base_addr+(1<<TARGET_SIZE_2));
1259   //cacheflush((void *)base_addr,(void *)base_addr+(1<<TARGET_SIZE_2),0);
1260   #endif
1261   #ifdef USE_MINI_HT
1262   memset(mini_ht,-1,sizeof(mini_ht));
1263   #endif
1264   // TLB
1265   for(page=0;page<0x100000;page++) {
1266     if(tlb_LUT_r[page]) {
1267       memory_map[page]=((tlb_LUT_r[page]&0xFFFFF000)-(page<<12)+(unsigned int)rdram-0x80000000)>>2;
1268       if(!tlb_LUT_w[page]||!invalid_code[page])
1269         memory_map[page]|=0x40000000; // Write protect
1270     }
1271     else memory_map[page]=-1;
1272     if(page==0x80000) page=0xC0000;
1273   }
1274   tlb_hacks();
1275 }
1276
1277 // Add an entry to jump_out after making a link
1278 void add_link(u_int vaddr,void *src)
1279 {
1280   u_int page=(vaddr^0x80000000)>>12;
1281   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[vaddr>>12]^0x80000000)>>12;
1282   if(page>4095) page=2048+(page&2047);
1283   inv_debug("add_link: %x -> %x (%d)\n",(int)src,vaddr,page);
1284   ll_add(jump_out+page,vaddr,src);
1285   //int ptr=get_pointer(src);
1286   //inv_debug("add_link: Pointer is to %x\n",(int)ptr);
1287 }
1288
1289 // If a code block was found to be unmodified (bit was set in
1290 // restore_candidate) and it remains unmodified (bit is clear
1291 // in invalid_code) then move the entries for that 4K page from
1292 // the dirty list to the clean list.
1293 void clean_blocks(u_int page)
1294 {
1295   struct ll_entry *head;
1296   inv_debug("INV: clean_blocks page=%d\n",page);
1297   head=jump_dirty[page];
1298   while(head!=NULL) {
1299     if(!invalid_code[head->vaddr>>12]) {
1300       // Don't restore blocks which are about to expire from the cache
1301       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1302         u_int start,end;
1303         if(verify_dirty(head->addr)) {
1304           //DebugMessage(M64MSG_VERBOSE, "Possibly Restore %x (%x)",head->vaddr, (int)head->addr);
1305           u_int i;
1306           u_int inv=0;
1307           get_bounds((int)head->addr,&start,&end);
1308           if(start-(u_int)rdram<0x800000) {
1309             for(i=(start-(u_int)rdram+0x80000000)>>12;i<=(end-1-(u_int)rdram+0x80000000)>>12;i++) {
1310               inv|=invalid_code[i];
1311             }
1312           }
1313           if((signed int)head->vaddr>=(signed int)0xC0000000) {
1314             u_int addr = (head->vaddr+(memory_map[head->vaddr>>12]<<2));
1315             //DebugMessage(M64MSG_VERBOSE, "addr=%x start=%x end=%x",addr,start,end);
1316             if(addr<start||addr>=end) inv=1;
1317           }
1318           else if((signed int)head->vaddr>=(signed int)0x80800000) {
1319             inv=1;
1320           }
1321           if(!inv) {
1322             void * clean_addr=(void *)get_clean_addr((int)head->addr);
1323             if((((u_int)clean_addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1324               u_int ppage=page;
1325               if(page<2048&&tlb_LUT_r[head->vaddr>>12]) ppage=(tlb_LUT_r[head->vaddr>>12]^0x80000000)>>12;
1326               inv_debug("INV: Restored %x (%x/%x)\n",head->vaddr, (int)head->addr, (int)clean_addr);
1327               //DebugMessage(M64MSG_VERBOSE, "page=%x, addr=%x",page,head->vaddr);
1328               //assert(head->vaddr>>12==(page|0x80000));
1329               ll_add_32(jump_in+ppage,head->vaddr,head->reg32,clean_addr);
1330               u_int *ht_bin=hash_table[((head->vaddr>>16)^head->vaddr)&0xFFFF];
1331               if(!head->reg32) {
1332                 if(ht_bin[0]==head->vaddr) {
1333                   ht_bin[1]=(int)clean_addr; // Replace existing entry
1334                 }
1335                 if(ht_bin[2]==head->vaddr) {
1336                   ht_bin[3]=(int)clean_addr; // Replace existing entry
1337                 }
1338               }
1339             }
1340           }
1341         }
1342       }
1343     }
1344     head=head->next;
1345   }
1346 }
1347
1348
1349 static void mov_alloc(struct regstat *current,int i)
1350 {
1351   // Note: Don't need to actually alloc the source registers
1352   if((~current->is32>>rs1[i])&1) {
1353     //alloc_reg64(current,i,rs1[i]);
1354     alloc_reg64(current,i,rt1[i]);
1355     current->is32&=~(1LL<<rt1[i]);
1356   } else {
1357     //alloc_reg(current,i,rs1[i]);
1358     alloc_reg(current,i,rt1[i]);
1359     current->is32|=(1LL<<rt1[i]);
1360   }
1361   clear_const(current,rs1[i]);
1362   clear_const(current,rt1[i]);
1363   dirty_reg(current,rt1[i]);
1364 }
1365
1366 static void shiftimm_alloc(struct regstat *current,int i)
1367 {
1368   clear_const(current,rs1[i]);
1369   clear_const(current,rt1[i]);
1370   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1371   {
1372     if(rt1[i]) {
1373       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1374       else lt1[i]=rs1[i];
1375       alloc_reg(current,i,rt1[i]);
1376       current->is32|=1LL<<rt1[i];
1377       dirty_reg(current,rt1[i]);
1378     }
1379   }
1380   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1381   {
1382     if(rt1[i]) {
1383       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1384       alloc_reg64(current,i,rt1[i]);
1385       current->is32&=~(1LL<<rt1[i]);
1386       dirty_reg(current,rt1[i]);
1387     }
1388   }
1389   if(opcode2[i]==0x3c) // DSLL32
1390   {
1391     if(rt1[i]) {
1392       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1393       alloc_reg64(current,i,rt1[i]);
1394       current->is32&=~(1LL<<rt1[i]);
1395       dirty_reg(current,rt1[i]);
1396     }
1397   }
1398   if(opcode2[i]==0x3e) // DSRL32
1399   {
1400     if(rt1[i]) {
1401       alloc_reg64(current,i,rs1[i]);
1402       if(imm[i]==32) {
1403         alloc_reg64(current,i,rt1[i]);
1404         current->is32&=~(1LL<<rt1[i]);
1405       } else {
1406         alloc_reg(current,i,rt1[i]);
1407         current->is32|=1LL<<rt1[i];
1408       }
1409       dirty_reg(current,rt1[i]);
1410     }
1411   }
1412   if(opcode2[i]==0x3f) // DSRA32
1413   {
1414     if(rt1[i]) {
1415       alloc_reg64(current,i,rs1[i]);
1416       alloc_reg(current,i,rt1[i]);
1417       current->is32|=1LL<<rt1[i];
1418       dirty_reg(current,rt1[i]);
1419     }
1420   }
1421 }
1422
1423 static void shift_alloc(struct regstat *current,int i)
1424 {
1425   if(rt1[i]) {
1426     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1427     {
1428       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1429       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1430       alloc_reg(current,i,rt1[i]);
1431       if(rt1[i]==rs2[i]) {
1432         alloc_reg_temp(current,i,-1);
1433         minimum_free_regs[i]=1;
1434       }
1435       current->is32|=1LL<<rt1[i];
1436     } else { // DSLLV/DSRLV/DSRAV
1437       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1438       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1439       alloc_reg64(current,i,rt1[i]);
1440       current->is32&=~(1LL<<rt1[i]);
1441       if(opcode2[i]==0x16||opcode2[i]==0x17) // DSRLV and DSRAV need a temporary register
1442       {
1443         alloc_reg_temp(current,i,-1);
1444         minimum_free_regs[i]=1;
1445       }
1446     }
1447     clear_const(current,rs1[i]);
1448     clear_const(current,rs2[i]);
1449     clear_const(current,rt1[i]);
1450     dirty_reg(current,rt1[i]);
1451   }
1452 }
1453
1454 static void alu_alloc(struct regstat *current,int i)
1455 {
1456   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1457     if(rt1[i]) {
1458       if(rs1[i]&&rs2[i]) {
1459         alloc_reg(current,i,rs1[i]);
1460         alloc_reg(current,i,rs2[i]);
1461       }
1462       else {
1463         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1464         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1465       }
1466       alloc_reg(current,i,rt1[i]);
1467     }
1468     current->is32|=1LL<<rt1[i];
1469   }
1470   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1471     if(rt1[i]) {
1472       if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1473       {
1474         alloc_reg64(current,i,rs1[i]);
1475         alloc_reg64(current,i,rs2[i]);
1476         alloc_reg(current,i,rt1[i]);
1477       } else {
1478         alloc_reg(current,i,rs1[i]);
1479         alloc_reg(current,i,rs2[i]);
1480         alloc_reg(current,i,rt1[i]);
1481       }
1482     }
1483     current->is32|=1LL<<rt1[i];
1484   }
1485   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1486     if(rt1[i]) {
1487       if(rs1[i]&&rs2[i]) {
1488         alloc_reg(current,i,rs1[i]);
1489         alloc_reg(current,i,rs2[i]);
1490       }
1491       else
1492       {
1493         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1494         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1495       }
1496       alloc_reg(current,i,rt1[i]);
1497       if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1498       {
1499         if(!((current->uu>>rt1[i])&1)) {
1500           alloc_reg64(current,i,rt1[i]);
1501         }
1502         if(get_reg(current->regmap,rt1[i]|64)>=0) {
1503           if(rs1[i]&&rs2[i]) {
1504             alloc_reg64(current,i,rs1[i]);
1505             alloc_reg64(current,i,rs2[i]);
1506           }
1507           else
1508           {
1509             // Is is really worth it to keep 64-bit values in registers?
1510             #ifdef NATIVE_64BIT
1511             if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg64(current,i,rs1[i]);
1512             if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg64(current,i,rs2[i]);
1513             #endif
1514           }
1515         }
1516         current->is32&=~(1LL<<rt1[i]);
1517       } else {
1518         current->is32|=1LL<<rt1[i];
1519       }
1520     }
1521   }
1522   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1523     if(rt1[i]) {
1524       if(rs1[i]&&rs2[i]) {
1525         if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1526           alloc_reg64(current,i,rs1[i]);
1527           alloc_reg64(current,i,rs2[i]);
1528           alloc_reg64(current,i,rt1[i]);
1529         } else {
1530           alloc_reg(current,i,rs1[i]);
1531           alloc_reg(current,i,rs2[i]);
1532           alloc_reg(current,i,rt1[i]);
1533         }
1534       }
1535       else {
1536         alloc_reg(current,i,rt1[i]);
1537         if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1538           // DADD used as move, or zeroing
1539           // If we have a 64-bit source, then make the target 64 bits too
1540           if(rs1[i]&&!((current->is32>>rs1[i])&1)) {
1541             if(get_reg(current->regmap,rs1[i])>=0) alloc_reg64(current,i,rs1[i]);
1542             alloc_reg64(current,i,rt1[i]);
1543           } else if(rs2[i]&&!((current->is32>>rs2[i])&1)) {
1544             if(get_reg(current->regmap,rs2[i])>=0) alloc_reg64(current,i,rs2[i]);
1545             alloc_reg64(current,i,rt1[i]);
1546           }
1547           if(opcode2[i]>=0x2e&&rs2[i]) {
1548             // DSUB used as negation - 64-bit result
1549             // If we have a 32-bit register, extend it to 64 bits
1550             if(get_reg(current->regmap,rs2[i])>=0) alloc_reg64(current,i,rs2[i]);
1551             alloc_reg64(current,i,rt1[i]);
1552           }
1553         }
1554       }
1555       if(rs1[i]&&rs2[i]) {
1556         current->is32&=~(1LL<<rt1[i]);
1557       } else if(rs1[i]) {
1558         current->is32&=~(1LL<<rt1[i]);
1559         if((current->is32>>rs1[i])&1)
1560           current->is32|=1LL<<rt1[i];
1561       } else if(rs2[i]) {
1562         current->is32&=~(1LL<<rt1[i]);
1563         if((current->is32>>rs2[i])&1)
1564           current->is32|=1LL<<rt1[i];
1565       } else {
1566         current->is32|=1LL<<rt1[i];
1567       }
1568     }
1569   }
1570   clear_const(current,rs1[i]);
1571   clear_const(current,rs2[i]);
1572   clear_const(current,rt1[i]);
1573   dirty_reg(current,rt1[i]);
1574 }
1575
1576 static void imm16_alloc(struct regstat *current,int i)
1577 {
1578   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1579   else lt1[i]=rs1[i];
1580   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1581   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1582     current->is32&=~(1LL<<rt1[i]);
1583     if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1584       // TODO: Could preserve the 32-bit flag if the immediate is zero
1585       alloc_reg64(current,i,rt1[i]);
1586       alloc_reg64(current,i,rs1[i]);
1587     }
1588     clear_const(current,rs1[i]);
1589     clear_const(current,rt1[i]);
1590   }
1591   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1592     if((~current->is32>>rs1[i])&1) alloc_reg64(current,i,rs1[i]);
1593     current->is32|=1LL<<rt1[i];
1594     clear_const(current,rs1[i]);
1595     clear_const(current,rt1[i]);
1596   }
1597   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1598     if(((~current->is32>>rs1[i])&1)&&opcode[i]>0x0c) {
1599       if(rs1[i]!=rt1[i]) {
1600         if(needed_again(rs1[i],i)) alloc_reg64(current,i,rs1[i]);
1601         alloc_reg64(current,i,rt1[i]);
1602         current->is32&=~(1LL<<rt1[i]);
1603       }
1604     }
1605     else current->is32|=1LL<<rt1[i]; // ANDI clears upper bits
1606     if(is_const(current,rs1[i])) {
1607       int v=get_const(current,rs1[i]);
1608       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1609       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1610       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1611     }
1612     else clear_const(current,rt1[i]);
1613   }
1614   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1615     if(is_const(current,rs1[i])) {
1616       int v=get_const(current,rs1[i]);
1617       set_const(current,rt1[i],v+imm[i]);
1618     }
1619     else clear_const(current,rt1[i]);
1620     current->is32|=1LL<<rt1[i];
1621   }
1622   else {
1623     set_const(current,rt1[i],((long long)((short)imm[i]))<<16); // LUI
1624     current->is32|=1LL<<rt1[i];
1625   }
1626   dirty_reg(current,rt1[i]);
1627 }
1628
1629 static void load_alloc(struct regstat *current,int i)
1630 {
1631   clear_const(current,rt1[i]);
1632   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1633   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1634   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1635   if(rt1[i]&&!((current->u>>rt1[i])&1)) {
1636     alloc_reg(current,i,rt1[i]);
1637     assert(get_reg(current->regmap,rt1[i])>=0);
1638     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1639     {
1640       current->is32&=~(1LL<<rt1[i]);
1641       alloc_reg64(current,i,rt1[i]);
1642     }
1643     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1644     {
1645       current->is32&=~(1LL<<rt1[i]);
1646       alloc_reg64(current,i,rt1[i]);
1647       alloc_all(current,i);
1648       alloc_reg64(current,i,FTEMP);
1649       minimum_free_regs[i]=HOST_REGS;
1650     }
1651     else current->is32|=1LL<<rt1[i];
1652     dirty_reg(current,rt1[i]);
1653     // If using TLB, need a register for pointer to the mapping table
1654     if(using_tlb) alloc_reg(current,i,TLREG);
1655     // LWL/LWR need a temporary register for the old value
1656     if(opcode[i]==0x22||opcode[i]==0x26)
1657     {
1658       alloc_reg(current,i,FTEMP);
1659       alloc_reg_temp(current,i,-1);
1660       minimum_free_regs[i]=1;
1661     }
1662   }
1663   else
1664   {
1665     // Load to r0 or unneeded register (dummy load)
1666     // but we still need a register to calculate the address
1667     if(opcode[i]==0x22||opcode[i]==0x26)
1668     {
1669       alloc_reg(current,i,FTEMP); // LWL/LWR need another temporary
1670     }
1671     // If using TLB, need a register for pointer to the mapping table
1672     if(using_tlb) alloc_reg(current,i,TLREG);
1673     alloc_reg_temp(current,i,-1);
1674     minimum_free_regs[i]=1;
1675     if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1676     {
1677       alloc_all(current,i);
1678       alloc_reg64(current,i,FTEMP);
1679       minimum_free_regs[i]=HOST_REGS;
1680     }
1681   }
1682 }
1683
1684 static void store_alloc(struct regstat *current,int i)
1685 {
1686   clear_const(current,rs2[i]);
1687   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1688   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1689   alloc_reg(current,i,rs2[i]);
1690   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1691     alloc_reg64(current,i,rs2[i]);
1692     if(rs2[i]) alloc_reg(current,i,FTEMP);
1693   }
1694   // If using TLB, need a register for pointer to the mapping table
1695   if(using_tlb) alloc_reg(current,i,TLREG);
1696   #if defined(HOST_IMM8)
1697   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1698   else alloc_reg(current,i,INVCP);
1699   #endif
1700   if(opcode[i]==0x2c||opcode[i]==0x2d) { // 64-bit SDL/SDR
1701     alloc_reg(current,i,FTEMP);
1702   }
1703   // We need a temporary register for address generation
1704   alloc_reg_temp(current,i,-1);
1705   minimum_free_regs[i]=1;
1706 }
1707
1708 static void c1ls_alloc(struct regstat *current,int i)
1709 {
1710   //clear_const(current,rs1[i]); // FIXME
1711   clear_const(current,rt1[i]);
1712   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1713   alloc_reg(current,i,CSREG); // Status
1714   alloc_reg(current,i,FTEMP);
1715   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1716     alloc_reg64(current,i,FTEMP);
1717   }
1718   // If using TLB, need a register for pointer to the mapping table
1719   if(using_tlb) alloc_reg(current,i,TLREG);
1720   #if defined(HOST_IMM8)
1721   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1722   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1723     alloc_reg(current,i,INVCP);
1724   #endif
1725   // We need a temporary register for address generation
1726   alloc_reg_temp(current,i,-1);
1727   minimum_free_regs[i]=1;
1728 }
1729
1730 #ifndef multdiv_alloc
1731 void multdiv_alloc(struct regstat *current,int i)
1732 {
1733   //  case 0x18: MULT
1734   //  case 0x19: MULTU
1735   //  case 0x1A: DIV
1736   //  case 0x1B: DIVU
1737   //  case 0x1C: DMULT
1738   //  case 0x1D: DMULTU
1739   //  case 0x1E: DDIV
1740   //  case 0x1F: DDIVU
1741   clear_const(current,rs1[i]);
1742   clear_const(current,rs2[i]);
1743   if(rs1[i]&&rs2[i])
1744   {
1745     if((opcode2[i]&4)==0) // 32-bit
1746     {
1747       current->u&=~(1LL<<HIREG);
1748       current->u&=~(1LL<<LOREG);
1749       alloc_reg(current,i,HIREG);
1750       alloc_reg(current,i,LOREG);
1751       alloc_reg(current,i,rs1[i]);
1752       alloc_reg(current,i,rs2[i]);
1753       current->is32|=1LL<<HIREG;
1754       current->is32|=1LL<<LOREG;
1755       dirty_reg(current,HIREG);
1756       dirty_reg(current,LOREG);
1757     }
1758     else // 64-bit
1759     {
1760       current->u&=~(1LL<<HIREG);
1761       current->u&=~(1LL<<LOREG);
1762       current->uu&=~(1LL<<HIREG);
1763       current->uu&=~(1LL<<LOREG);
1764       alloc_reg64(current,i,HIREG);
1765       //if(HOST_REGS>10) alloc_reg64(current,i,LOREG);  //*SEB* Why commenting this line? uncommenting make SM64 freeze after title (before mario head and spinning stars)
1766       alloc_reg64(current,i,rs1[i]);
1767       alloc_reg64(current,i,rs2[i]);
1768       alloc_all(current,i);
1769       current->is32&=~(1LL<<HIREG);
1770       current->is32&=~(1LL<<LOREG);
1771       dirty_reg(current,HIREG);
1772       dirty_reg(current,LOREG);
1773       minimum_free_regs[i]=HOST_REGS;
1774     }
1775   }
1776   else
1777   {
1778     // Multiply by zero is zero.
1779     // MIPS does not have a divide by zero exception.
1780     // The result is undefined, we return zero.
1781         alloc_reg(current,i,HIREG);
1782         alloc_reg(current,i,LOREG);
1783         current->is32|=1LL<<HIREG;
1784         current->is32|=1LL<<LOREG;
1785         dirty_reg(current,HIREG);
1786         dirty_reg(current,LOREG);
1787   }
1788 }
1789 #endif
1790
1791 static void cop0_alloc(struct regstat *current,int i)
1792 {
1793   if(opcode2[i]==0) // MFC0
1794   {
1795     if(rt1[i]) {
1796       clear_const(current,rt1[i]);
1797       alloc_all(current,i);
1798       alloc_reg(current,i,rt1[i]);
1799       current->is32|=1LL<<rt1[i];
1800       dirty_reg(current,rt1[i]);
1801     }
1802   }
1803   else if(opcode2[i]==4) // MTC0
1804   {
1805     if(rs1[i]){
1806       clear_const(current,rs1[i]);
1807       alloc_reg(current,i,rs1[i]);
1808       alloc_all(current,i);
1809     }
1810     else {
1811       alloc_all(current,i); // FIXME: Keep r0
1812       current->u&=~1LL;
1813       alloc_reg(current,i,0);
1814     }
1815   }
1816   else
1817   {
1818     // TLBR/TLBWI/TLBWR/TLBP/ERET
1819     assert(opcode2[i]==0x10);
1820     alloc_all(current,i);
1821   }
1822   minimum_free_regs[i]=HOST_REGS;
1823 }
1824
1825 static void cop1_alloc(struct regstat *current,int i)
1826 {
1827   alloc_reg(current,i,CSREG); // Load status
1828   if(opcode2[i]<3) // MFC1/DMFC1/CFC1
1829   {
1830     assert(rt1[i]);
1831     clear_const(current,rt1[i]);
1832     if(opcode2[i]==1) {
1833       alloc_reg64(current,i,rt1[i]); // DMFC1
1834       current->is32&=~(1LL<<rt1[i]);
1835     }else{
1836       alloc_reg(current,i,rt1[i]); // MFC1/CFC1
1837       current->is32|=1LL<<rt1[i];
1838     }
1839     dirty_reg(current,rt1[i]);
1840     alloc_reg_temp(current,i,-1);
1841   }
1842   else if(opcode2[i]>3) // MTC1/DMTC1/CTC1
1843   {
1844     if(rs1[i]){
1845       clear_const(current,rs1[i]);
1846       if(opcode2[i]==5)
1847         alloc_reg64(current,i,rs1[i]); // DMTC1
1848       else
1849         alloc_reg(current,i,rs1[i]); // MTC1/CTC1
1850       alloc_reg_temp(current,i,-1);
1851     }
1852     else {
1853       current->u&=~1LL;
1854       alloc_reg(current,i,0);
1855       alloc_reg_temp(current,i,-1);
1856     }
1857   }
1858   minimum_free_regs[i]=1;
1859 }
1860 static void fconv_alloc(struct regstat *current,int i)
1861 {
1862   alloc_reg(current,i,CSREG); // Load status
1863   alloc_reg_temp(current,i,-1);
1864   minimum_free_regs[i]=1;
1865 }
1866 static void float_alloc(struct regstat *current,int i)
1867 {
1868   alloc_reg(current,i,CSREG); // Load status
1869   alloc_reg_temp(current,i,-1);
1870   minimum_free_regs[i]=1;
1871 }
1872 static void fcomp_alloc(struct regstat *current,int i)
1873 {
1874   alloc_reg(current,i,CSREG); // Load status
1875   alloc_reg(current,i,FSREG); // Load flags
1876   dirty_reg(current,FSREG); // Flag will be modified
1877   alloc_reg_temp(current,i,-1);
1878   minimum_free_regs[i]=1;
1879 }
1880
1881 static void syscall_alloc(struct regstat *current,int i)
1882 {
1883   alloc_cc(current,i);
1884   dirty_reg(current,CCREG);
1885   alloc_all(current,i);
1886   minimum_free_regs[i]=HOST_REGS;
1887   current->isconst=0;
1888 }
1889
1890 static void delayslot_alloc(struct regstat *current,int i)
1891 {
1892   switch(itype[i]) {
1893     case UJUMP:
1894     case CJUMP:
1895     case SJUMP:
1896     case RJUMP:
1897     case FJUMP:
1898     case SYSCALL:
1899     case SPAN:
1900       assem_debug("jump in the delay slot.  this shouldn't happen.");//exit(1);
1901       DebugMessage(M64MSG_VERBOSE, "Disabled speculative precompilation");
1902       stop_after_jal=1;
1903       break;
1904     case IMM16:
1905       imm16_alloc(current,i);
1906       break;
1907     case LOAD:
1908     case LOADLR:
1909       load_alloc(current,i);
1910       break;
1911     case STORE:
1912     case STORELR:
1913       store_alloc(current,i);
1914       break;
1915     case ALU:
1916       alu_alloc(current,i);
1917       break;
1918     case SHIFT:
1919       shift_alloc(current,i);
1920       break;
1921     case MULTDIV:
1922       multdiv_alloc(current,i);
1923       break;
1924     case SHIFTIMM:
1925       shiftimm_alloc(current,i);
1926       break;
1927     case MOV:
1928       mov_alloc(current,i);
1929       break;
1930     case COP0:
1931       cop0_alloc(current,i);
1932       break;
1933     case COP1:
1934       cop1_alloc(current,i);
1935       break;
1936     case C1LS:
1937       c1ls_alloc(current,i);
1938       break;
1939     case FCONV:
1940       fconv_alloc(current,i);
1941       break;
1942     case FLOAT:
1943       float_alloc(current,i);
1944       break;
1945     case FCOMP:
1946       fcomp_alloc(current,i);
1947       break;
1948   }
1949 }
1950
1951 // Special case where a branch and delay slot span two pages in virtual memory
1952 static void pagespan_alloc(struct regstat *current,int i)
1953 {
1954   current->isconst=0;
1955   current->wasconst=0;
1956   regs[i].wasconst=0;
1957   minimum_free_regs[i]=HOST_REGS;
1958   alloc_all(current,i);
1959   alloc_cc(current,i);
1960   dirty_reg(current,CCREG);
1961   if(opcode[i]==3) // JAL
1962   {
1963     alloc_reg(current,i,31);
1964     dirty_reg(current,31);
1965   }
1966   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
1967   {
1968     alloc_reg(current,i,rs1[i]);
1969     if (rt1[i]!=0) {
1970       alloc_reg(current,i,rt1[i]);
1971       dirty_reg(current,rt1[i]);
1972     }
1973   }
1974   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
1975   {
1976     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1977     if(rs2[i]) alloc_reg(current,i,rs2[i]);
1978     if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1979     {
1980       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1981       if(rs2[i]) alloc_reg64(current,i,rs2[i]);
1982     }
1983   }
1984   else
1985   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
1986   {
1987     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1988     if(!((current->is32>>rs1[i])&1))
1989     {
1990       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1991     }
1992   }
1993   else
1994   if(opcode[i]==0x11) // BC1
1995   {
1996     alloc_reg(current,i,FSREG);
1997     alloc_reg(current,i,CSREG);
1998   }
1999   //else ...
2000 }
2001
2002 static void add_stub(int type,int addr,int retaddr,int a,int b,int c,int d,int e)
2003 {
2004   stubs[stubcount][0]=type;
2005   stubs[stubcount][1]=addr;
2006   stubs[stubcount][2]=retaddr;
2007   stubs[stubcount][3]=a;
2008   stubs[stubcount][4]=b;
2009   stubs[stubcount][5]=c;
2010   stubs[stubcount][6]=d;
2011   stubs[stubcount][7]=e;
2012   stubcount++;
2013 }
2014
2015 // Write out a single register
2016 static void wb_register(signed char r,signed char regmap[],uint64_t dirty,uint64_t is32)
2017 {
2018   int hr;
2019   for(hr=0;hr<HOST_REGS;hr++) {
2020     if(hr!=EXCLUDE_REG) {
2021       if((regmap[hr]&63)==r) {
2022         if((dirty>>hr)&1) {
2023           if(regmap[hr]<64) {
2024             emit_storereg(r,hr);
2025             if((is32>>regmap[hr])&1) {
2026               emit_sarimm(hr,31,hr);
2027               emit_storereg(r|64,hr);
2028             }
2029           }else{
2030             emit_storereg(r|64,hr);
2031           }
2032         }
2033       }
2034     }
2035   }
2036 }
2037 #if 0
2038 static int mchecksum()
2039 {
2040   //if(!tracedebug) return 0;
2041   int i;
2042   int sum=0;
2043   for(i=0;i<2097152;i++) {
2044     unsigned int temp=sum;
2045     sum<<=1;
2046     sum|=(~temp)>>31;
2047     sum^=((u_int *)rdram)[i];
2048   }
2049   return sum;
2050 }
2051
2052 static int rchecksum()
2053 {
2054   int i;
2055   int sum=0;
2056   for(i=0;i<64;i++)
2057     sum^=((u_int *)reg)[i];
2058   return sum;
2059 }
2060
2061 static void rlist()
2062 {
2063   int i;
2064   DebugMessage(M64MSG_VERBOSE, "TRACE: ");
2065   for(i=0;i<32;i++)
2066     DebugMessage(M64MSG_VERBOSE, "r%d:%8x%8x ",i,((int *)(reg+i))[1],((int *)(reg+i))[0]);
2067   DebugMessage(M64MSG_VERBOSE, "TRACE: ");
2068   for(i=0;i<32;i++)
2069     DebugMessage(M64MSG_VERBOSE, "f%d:%8x%8x ",i,((int*)reg_cop1_simple[i])[1],*((int*)reg_cop1_simple[i]));
2070 }
2071
2072 static void enabletrace()
2073 {
2074   tracedebug=1;
2075 }
2076
2077
2078 static void memdebug(int i)
2079 {
2080   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (checksum %x) lo=%8x%8x",Count,next_interupt,mchecksum(),(int)(reg[LOREG]>>32),(int)reg[LOREG]);
2081   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (rchecksum %x)",Count,next_interupt,rchecksum());
2082   //rlist();
2083   //if(tracedebug) {
2084   //if(Count>=-2084597794) {
2085   if((signed int)Count>=-2084597794&&(signed int)Count<0) {
2086   //if(0) {
2087     DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (checksum %x)",Count,next_interupt,mchecksum());
2088     //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (checksum %x) Status=%x",Count,next_interupt,mchecksum(),Status);
2089     //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (checksum %x) hi=%8x%8x",Count,next_interupt,mchecksum(),(int)(reg[HIREG]>>32),(int)reg[HIREG]);
2090     rlist();
2091     #if NEW_DYNAREC == NEW_DYNAREC_X86
2092     DebugMessage(M64MSG_VERBOSE, "TRACE: %x",(&i)[-1]);
2093     #endif
2094     #if NEW_DYNAREC == NEW_DYNAREC_ARM
2095     int j;
2096     DebugMessage(M64MSG_VERBOSE, "TRACE: %x ",(&j)[10]);
2097     DebugMessage(M64MSG_VERBOSE, "TRACE: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",(&j)[1],(&j)[2],(&j)[3],(&j)[4],(&j)[5],(&j)[6],(&j)[7],(&j)[8],(&j)[9],(&j)[10],(&j)[11],(&j)[12],(&j)[13],(&j)[14],(&j)[15],(&j)[16],(&j)[17],(&j)[18],(&j)[19],(&j)[20]);
2098     #endif
2099     //fflush(stdout);
2100   }
2101   //DebugMessage(M64MSG_VERBOSE, "TRACE: %x",(&i)[-1]);
2102 }
2103 #endif
2104
2105 /* Debug:
2106 static void tlb_debug(u_int cause, u_int addr, u_int iaddr)
2107 {
2108   DebugMessage(M64MSG_VERBOSE, "TLB Exception: instruction=%x addr=%x cause=%x",iaddr, addr, cause);
2109 }
2110 end debug */
2111
2112 static void alu_assemble(int i,struct regstat *i_regs)
2113 {
2114   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2115     if(rt1[i]) {
2116       signed char s1,s2,t;
2117       t=get_reg(i_regs->regmap,rt1[i]);
2118       if(t>=0) {
2119         s1=get_reg(i_regs->regmap,rs1[i]);
2120         s2=get_reg(i_regs->regmap,rs2[i]);
2121         if(rs1[i]&&rs2[i]) {
2122           assert(s1>=0);
2123           assert(s2>=0);
2124           if(opcode2[i]&2) emit_sub(s1,s2,t);
2125           else emit_add(s1,s2,t);
2126         }
2127         else if(rs1[i]) {
2128           if(s1>=0) emit_mov(s1,t);
2129           else emit_loadreg(rs1[i],t);
2130         }
2131         else if(rs2[i]) {
2132           if(s2>=0) {
2133             if(opcode2[i]&2) emit_neg(s2,t);
2134             else emit_mov(s2,t);
2135           }
2136           else {
2137             emit_loadreg(rs2[i],t);
2138             if(opcode2[i]&2) emit_neg(t,t);
2139           }
2140         }
2141         else emit_zeroreg(t);
2142       }
2143     }
2144   }
2145   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2146     if(rt1[i]) {
2147       signed char s1l,s2l,s1h,s2h,tl,th;
2148       tl=get_reg(i_regs->regmap,rt1[i]);
2149       th=get_reg(i_regs->regmap,rt1[i]|64);
2150       if(tl>=0) {
2151         s1l=get_reg(i_regs->regmap,rs1[i]);
2152         s2l=get_reg(i_regs->regmap,rs2[i]);
2153         s1h=get_reg(i_regs->regmap,rs1[i]|64);
2154         s2h=get_reg(i_regs->regmap,rs2[i]|64);
2155         if(rs1[i]&&rs2[i]) {
2156           assert(s1l>=0);
2157           assert(s2l>=0);
2158           if(opcode2[i]&2) emit_subs(s1l,s2l,tl);
2159           else emit_adds(s1l,s2l,tl);
2160           if(th>=0) {
2161             #ifdef INVERTED_CARRY
2162             if(opcode2[i]&2) {if(s1h!=th) emit_mov(s1h,th);emit_sbb(th,s2h);}
2163             #else
2164             if(opcode2[i]&2) emit_sbc(s1h,s2h,th);
2165             #endif
2166             else emit_add(s1h,s2h,th);
2167           }
2168         }
2169         else if(rs1[i]) {
2170           if(s1l>=0) emit_mov(s1l,tl);
2171           else emit_loadreg(rs1[i],tl);
2172           if(th>=0) {
2173             if(s1h>=0) emit_mov(s1h,th);
2174             else emit_loadreg(rs1[i]|64,th);
2175           }
2176         }
2177         else if(rs2[i]) {
2178           if(s2l>=0) {
2179             if(opcode2[i]&2) emit_negs(s2l,tl);
2180             else emit_mov(s2l,tl);
2181           }
2182           else {
2183             emit_loadreg(rs2[i],tl);
2184             if(opcode2[i]&2) emit_negs(tl,tl);
2185           }
2186           if(th>=0) {
2187             #ifdef INVERTED_CARRY
2188             if(s2h>=0) emit_mov(s2h,th);
2189             else emit_loadreg(rs2[i]|64,th);
2190             if(opcode2[i]&2) {
2191               emit_adcimm(-1,th); // x86 has inverted carry flag
2192               emit_not(th,th);
2193             }
2194             #else
2195             if(opcode2[i]&2) {
2196               if(s2h>=0) emit_rscimm(s2h,0,th);
2197               else {
2198                 emit_loadreg(rs2[i]|64,th);
2199                 emit_rscimm(th,0,th);
2200               }
2201             }else{
2202               if(s2h>=0) emit_mov(s2h,th);
2203               else emit_loadreg(rs2[i]|64,th);
2204             }
2205             #endif
2206           }
2207         }
2208         else {
2209           emit_zeroreg(tl);
2210           if(th>=0) emit_zeroreg(th);
2211         }
2212       }
2213     }
2214   }
2215   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2216     if(rt1[i]) {
2217       signed char s1l,s1h,s2l,s2h,t;
2218       if(!((i_regs->was32>>rs1[i])&(i_regs->was32>>rs2[i])&1))
2219       {
2220         t=get_reg(i_regs->regmap,rt1[i]);
2221         //assert(t>=0);
2222         if(t>=0) {
2223           s1l=get_reg(i_regs->regmap,rs1[i]);
2224           s1h=get_reg(i_regs->regmap,rs1[i]|64);
2225           s2l=get_reg(i_regs->regmap,rs2[i]);
2226           s2h=get_reg(i_regs->regmap,rs2[i]|64);
2227           if(rs2[i]==0) // rx<r0
2228           {
2229             assert(s1h>=0);
2230             if(opcode2[i]==0x2a) // SLT
2231               emit_shrimm(s1h,31,t);
2232             else // SLTU (unsigned can not be less than zero)
2233               emit_zeroreg(t);
2234           }
2235           else if(rs1[i]==0) // r0<rx
2236           {
2237             assert(s2h>=0);
2238             if(opcode2[i]==0x2a) // SLT
2239               emit_set_gz64_32(s2h,s2l,t);
2240             else // SLTU (set if not zero)
2241               emit_set_nz64_32(s2h,s2l,t);
2242           }
2243           else {
2244             assert(s1l>=0);assert(s1h>=0);
2245             assert(s2l>=0);assert(s2h>=0);
2246             if(opcode2[i]==0x2a) // SLT
2247               emit_set_if_less64_32(s1h,s1l,s2h,s2l,t);
2248             else // SLTU
2249               emit_set_if_carry64_32(s1h,s1l,s2h,s2l,t);
2250           }
2251         }
2252       } else {
2253         t=get_reg(i_regs->regmap,rt1[i]);
2254         //assert(t>=0);
2255         if(t>=0) {
2256           s1l=get_reg(i_regs->regmap,rs1[i]);
2257           s2l=get_reg(i_regs->regmap,rs2[i]);
2258           if(rs2[i]==0) // rx<r0
2259           {
2260             assert(s1l>=0);
2261             if(opcode2[i]==0x2a) // SLT
2262               emit_shrimm(s1l,31,t);
2263             else // SLTU (unsigned can not be less than zero)
2264               emit_zeroreg(t);
2265           }
2266           else if(rs1[i]==0) // r0<rx
2267           {
2268             assert(s2l>=0);
2269             if(opcode2[i]==0x2a) // SLT
2270               emit_set_gz32(s2l,t);
2271             else // SLTU (set if not zero)
2272               emit_set_nz32(s2l,t);
2273           }
2274           else{
2275             assert(s1l>=0);assert(s2l>=0);
2276             if(opcode2[i]==0x2a) // SLT
2277               emit_set_if_less32(s1l,s2l,t);
2278             else // SLTU
2279               emit_set_if_carry32(s1l,s2l,t);
2280           }
2281         }
2282       }
2283     }
2284   }
2285   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2286     if(rt1[i]) {
2287       signed char s1l,s1h,s2l,s2h,th,tl;
2288       tl=get_reg(i_regs->regmap,rt1[i]);
2289       th=get_reg(i_regs->regmap,rt1[i]|64);
2290       if(!((i_regs->was32>>rs1[i])&(i_regs->was32>>rs2[i])&1)&&th>=0)
2291       {
2292         assert(tl>=0);
2293         if(tl>=0) {
2294           s1l=get_reg(i_regs->regmap,rs1[i]);
2295           s1h=get_reg(i_regs->regmap,rs1[i]|64);
2296           s2l=get_reg(i_regs->regmap,rs2[i]);
2297           s2h=get_reg(i_regs->regmap,rs2[i]|64);
2298           if(rs1[i]&&rs2[i]) {
2299             assert(s1l>=0);assert(s1h>=0);
2300             assert(s2l>=0);assert(s2h>=0);
2301             if(opcode2[i]==0x24) { // AND
2302               emit_and(s1l,s2l,tl);
2303               emit_and(s1h,s2h,th);
2304             } else
2305             if(opcode2[i]==0x25) { // OR
2306               emit_or(s1l,s2l,tl);
2307               emit_or(s1h,s2h,th);
2308             } else
2309             if(opcode2[i]==0x26) { // XOR
2310               emit_xor(s1l,s2l,tl);
2311               emit_xor(s1h,s2h,th);
2312             } else
2313             if(opcode2[i]==0x27) { // NOR
2314               emit_or(s1l,s2l,tl);
2315               emit_or(s1h,s2h,th);
2316               emit_not(tl,tl);
2317               emit_not(th,th);
2318             }
2319           }
2320           else
2321           {
2322             if(opcode2[i]==0x24) { // AND
2323               emit_zeroreg(tl);
2324               emit_zeroreg(th);
2325             } else
2326             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2327               if(rs1[i]){
2328                 if(s1l>=0) emit_mov(s1l,tl);
2329                 else emit_loadreg(rs1[i],tl);
2330                 if(s1h>=0) emit_mov(s1h,th);
2331                 else emit_loadreg(rs1[i]|64,th);
2332               }
2333               else
2334               if(rs2[i]){
2335                 if(s2l>=0) emit_mov(s2l,tl);
2336                 else emit_loadreg(rs2[i],tl);
2337                 if(s2h>=0) emit_mov(s2h,th);
2338                 else emit_loadreg(rs2[i]|64,th);
2339               }
2340               else{
2341                 emit_zeroreg(tl);
2342                 emit_zeroreg(th);
2343               }
2344             } else
2345             if(opcode2[i]==0x27) { // NOR
2346               if(rs1[i]){
2347                 if(s1l>=0) emit_not(s1l,tl);
2348                 else{
2349                   emit_loadreg(rs1[i],tl);
2350                   emit_not(tl,tl);
2351                 }
2352                 if(s1h>=0) emit_not(s1h,th);
2353                 else{
2354                   emit_loadreg(rs1[i]|64,th);
2355                   emit_not(th,th);
2356                 }
2357               }
2358               else
2359               if(rs2[i]){
2360                 if(s2l>=0) emit_not(s2l,tl);
2361                 else{
2362                   emit_loadreg(rs2[i],tl);
2363                   emit_not(tl,tl);
2364                 }
2365                 if(s2h>=0) emit_not(s2h,th);
2366                 else{
2367                   emit_loadreg(rs2[i]|64,th);
2368                   emit_not(th,th);
2369                 }
2370               }
2371               else {
2372                 emit_movimm(-1,tl);
2373                 emit_movimm(-1,th);
2374               }
2375             }
2376           }
2377         }
2378       }
2379       else
2380       {
2381         // 32 bit
2382         if(tl>=0) {
2383           s1l=get_reg(i_regs->regmap,rs1[i]);
2384           s2l=get_reg(i_regs->regmap,rs2[i]);
2385           if(rs1[i]&&rs2[i]) {
2386             assert(s1l>=0);
2387             assert(s2l>=0);
2388             if(opcode2[i]==0x24) { // AND
2389               emit_and(s1l,s2l,tl);
2390             } else
2391             if(opcode2[i]==0x25) { // OR
2392               emit_or(s1l,s2l,tl);
2393             } else
2394             if(opcode2[i]==0x26) { // XOR
2395               emit_xor(s1l,s2l,tl);
2396             } else
2397             if(opcode2[i]==0x27) { // NOR
2398               emit_or(s1l,s2l,tl);
2399               emit_not(tl,tl);
2400             }
2401           }
2402           else
2403           {
2404             if(opcode2[i]==0x24) { // AND
2405               emit_zeroreg(tl);
2406             } else
2407             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2408               if(rs1[i]){
2409                 if(s1l>=0) emit_mov(s1l,tl);
2410                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2411               }
2412               else
2413               if(rs2[i]){
2414                 if(s2l>=0) emit_mov(s2l,tl);
2415                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2416               }
2417               else emit_zeroreg(tl);
2418             } else
2419             if(opcode2[i]==0x27) { // NOR
2420               if(rs1[i]){
2421                 if(s1l>=0) emit_not(s1l,tl);
2422                 else {
2423                   emit_loadreg(rs1[i],tl);
2424                   emit_not(tl,tl);
2425                 }
2426               }
2427               else
2428               if(rs2[i]){
2429                 if(s2l>=0) emit_not(s2l,tl);
2430                 else {
2431                   emit_loadreg(rs2[i],tl);
2432                   emit_not(tl,tl);
2433                 }
2434               }
2435               else emit_movimm(-1,tl);
2436             }
2437           }
2438         }
2439       }
2440     }
2441   }
2442 }
2443
2444 static void imm16_assemble(int i,struct regstat *i_regs)
2445 {
2446   if (opcode[i]==0x0f) { // LUI
2447     if(rt1[i]) {
2448       signed char t;
2449       t=get_reg(i_regs->regmap,rt1[i]);
2450       //assert(t>=0);
2451       if(t>=0) {
2452         if(!((i_regs->isconst>>t)&1))
2453           emit_movimm(imm[i]<<16,t);
2454       }
2455     }
2456   }
2457   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2458     if(rt1[i]) {
2459       signed char s,t;
2460       t=get_reg(i_regs->regmap,rt1[i]);
2461       s=get_reg(i_regs->regmap,rs1[i]);
2462       if(rs1[i]) {
2463         //assert(t>=0);
2464         //assert(s>=0);
2465         if(t>=0) {
2466           if(!((i_regs->isconst>>t)&1)) {
2467             if(s<0) {
2468               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2469               emit_addimm(t,imm[i],t);
2470             }else{
2471               if(!((i_regs->wasconst>>s)&1))
2472                 emit_addimm(s,imm[i],t);
2473               else
2474                 emit_movimm(constmap[i][s]+imm[i],t);
2475             }
2476           }
2477         }
2478       } else {
2479         if(t>=0) {
2480           if(!((i_regs->isconst>>t)&1))
2481             emit_movimm(imm[i],t);
2482         }
2483       }
2484     }
2485   }
2486   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2487     if(rt1[i]) {
2488       signed char sh,sl,th,tl;
2489       th=get_reg(i_regs->regmap,rt1[i]|64);
2490       tl=get_reg(i_regs->regmap,rt1[i]);
2491       sh=get_reg(i_regs->regmap,rs1[i]|64);
2492       sl=get_reg(i_regs->regmap,rs1[i]);
2493       if(tl>=0) {
2494         if(rs1[i]) {
2495           assert(sh>=0);
2496           assert(sl>=0);
2497           if(th>=0) {
2498             emit_addimm64_32(sh,sl,imm[i],th,tl);
2499           }
2500           else {
2501             emit_addimm(sl,imm[i],tl);
2502           }
2503         } else {
2504           emit_movimm(imm[i],tl);
2505           if(th>=0) emit_movimm(((signed int)imm[i])>>31,th);
2506         }
2507       }
2508     }
2509   }
2510   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2511     if(rt1[i]) {
2512       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2513       signed char sh,sl,t;
2514       t=get_reg(i_regs->regmap,rt1[i]);
2515       sh=get_reg(i_regs->regmap,rs1[i]|64);
2516       sl=get_reg(i_regs->regmap,rs1[i]);
2517       //assert(t>=0);
2518       if(t>=0) {
2519         if(rs1[i]>0) {
2520           if(sh<0) assert((i_regs->was32>>rs1[i])&1);
2521           if(sh<0||((i_regs->was32>>rs1[i])&1)) {
2522             if(opcode[i]==0x0a) { // SLTI
2523               if(sl<0) {
2524                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2525                 emit_slti32(t,imm[i],t);
2526               }else{
2527                 emit_slti32(sl,imm[i],t);
2528               }
2529             }
2530             else { // SLTIU
2531               if(sl<0) {
2532                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2533                 emit_sltiu32(t,imm[i],t);
2534               }else{
2535                 emit_sltiu32(sl,imm[i],t);
2536               }
2537             }
2538           }else{ // 64-bit
2539             assert(sl>=0);
2540             if(opcode[i]==0x0a) // SLTI
2541               emit_slti64_32(sh,sl,imm[i],t);
2542             else // SLTIU
2543               emit_sltiu64_32(sh,sl,imm[i],t);
2544           }
2545         }else{
2546           // SLTI(U) with r0 is just stupid,
2547           // nonetheless examples can be found
2548           if(opcode[i]==0x0a) // SLTI
2549             if(0<imm[i]) emit_movimm(1,t);
2550             else emit_zeroreg(t);
2551           else // SLTIU
2552           {
2553             if(imm[i]) emit_movimm(1,t);
2554             else emit_zeroreg(t);
2555           }
2556         }
2557       }
2558     }
2559   }
2560   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2561     if(rt1[i]) {
2562       signed char sh,sl,th,tl;
2563       th=get_reg(i_regs->regmap,rt1[i]|64);
2564       tl=get_reg(i_regs->regmap,rt1[i]);
2565       sh=get_reg(i_regs->regmap,rs1[i]|64);
2566       sl=get_reg(i_regs->regmap,rs1[i]);
2567       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2568         if(opcode[i]==0x0c) //ANDI
2569         {
2570           if(rs1[i]) {
2571             if(sl<0) {
2572               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2573               emit_andimm(tl,imm[i],tl);
2574             }else{
2575               if(!((i_regs->wasconst>>sl)&1))
2576                 emit_andimm(sl,imm[i],tl);
2577               else
2578                 emit_movimm(constmap[i][sl]&imm[i],tl);
2579             }
2580           }
2581           else
2582             emit_zeroreg(tl);
2583           if(th>=0) emit_zeroreg(th);
2584         }
2585         else
2586         {
2587           if(rs1[i]) {
2588             if(sl<0) {
2589               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2590             }
2591             if(th>=0) {
2592               if(sh<0) {
2593                 emit_loadreg(rs1[i]|64,th);
2594               }else{
2595                 emit_mov(sh,th);
2596               }
2597             }
2598             if(opcode[i]==0x0d) { //ORI
2599             if(sl<0) {
2600               emit_orimm(tl,imm[i],tl);
2601             }else{
2602               if(!((i_regs->wasconst>>sl)&1))
2603                 emit_orimm(sl,imm[i],tl);
2604               else
2605                 emit_movimm(constmap[i][sl]|imm[i],tl);
2606             }
2607             }
2608             if(opcode[i]==0x0e) { //XORI
2609             if(sl<0) {
2610               emit_xorimm(tl,imm[i],tl);
2611             }else{
2612               if(!((i_regs->wasconst>>sl)&1))
2613                 emit_xorimm(sl,imm[i],tl);
2614               else
2615                 emit_movimm(constmap[i][sl]^imm[i],tl);
2616             }
2617             }
2618           }
2619           else {
2620             emit_movimm(imm[i],tl);
2621             if(th>=0) emit_zeroreg(th);
2622           }
2623         }
2624       }
2625     }
2626   }
2627 }
2628
2629 static void shiftimm_assemble(int i,struct regstat *i_regs)
2630 {
2631   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2632   {
2633     if(rt1[i]) {
2634       signed char s,t;
2635       t=get_reg(i_regs->regmap,rt1[i]);
2636       s=get_reg(i_regs->regmap,rs1[i]);
2637       //assert(t>=0);
2638       if(t>=0){
2639         if(rs1[i]==0)
2640         {
2641           emit_zeroreg(t);
2642         }
2643         else
2644         {
2645           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2646           if(imm[i]) {
2647             if(opcode2[i]==0) // SLL
2648             {
2649               emit_shlimm(s<0?t:s,imm[i],t);
2650             }
2651             if(opcode2[i]==2) // SRL
2652             {
2653               emit_shrimm(s<0?t:s,imm[i],t);
2654             }
2655             if(opcode2[i]==3) // SRA
2656             {
2657               emit_sarimm(s<0?t:s,imm[i],t);
2658             }
2659           }else{
2660             // Shift by zero
2661             if(s>=0 && s!=t) emit_mov(s,t);
2662           }
2663         }
2664       }
2665       //emit_storereg(rt1[i],t); //DEBUG
2666     }
2667   }
2668   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2669   {
2670     if(rt1[i]) {
2671       signed char sh,sl,th,tl;
2672       th=get_reg(i_regs->regmap,rt1[i]|64);
2673       tl=get_reg(i_regs->regmap,rt1[i]);
2674       sh=get_reg(i_regs->regmap,rs1[i]|64);
2675       sl=get_reg(i_regs->regmap,rs1[i]);
2676       if(tl>=0) {
2677         if(rs1[i]==0)
2678         {
2679           emit_zeroreg(tl);
2680           if(th>=0) emit_zeroreg(th);
2681         }
2682         else
2683         {
2684           assert(sl>=0);
2685           assert(sh>=0);
2686           if(imm[i]) {
2687             if(opcode2[i]==0x38) // DSLL
2688             {
2689               if(th>=0) emit_shldimm(sh,sl,imm[i],th);
2690               emit_shlimm(sl,imm[i],tl);
2691             }
2692             if(opcode2[i]==0x3a) // DSRL
2693             {
2694               emit_shrdimm(sl,sh,imm[i],tl);
2695               if(th>=0) emit_shrimm(sh,imm[i],th);
2696             }
2697             if(opcode2[i]==0x3b) // DSRA
2698             {
2699               emit_shrdimm(sl,sh,imm[i],tl);
2700               if(th>=0) emit_sarimm(sh,imm[i],th);
2701             }
2702           }else{
2703             // Shift by zero
2704             if(sl!=tl) emit_mov(sl,tl);
2705             if(th>=0&&sh!=th) emit_mov(sh,th);
2706           }
2707         }
2708       }
2709     }
2710   }
2711   if(opcode2[i]==0x3c) // DSLL32
2712   {
2713     if(rt1[i]) {
2714       signed char sl,tl,th;
2715       tl=get_reg(i_regs->regmap,rt1[i]);
2716       th=get_reg(i_regs->regmap,rt1[i]|64);
2717       sl=get_reg(i_regs->regmap,rs1[i]);
2718       if(th>=0||tl>=0){
2719         assert(tl>=0);
2720         assert(th>=0);
2721         assert(sl>=0);
2722         emit_mov(sl,th);
2723         emit_zeroreg(tl);
2724         if(imm[i]>32)
2725         {
2726           emit_shlimm(th,imm[i]&31,th);
2727         }
2728       }
2729     }
2730   }
2731   if(opcode2[i]==0x3e) // DSRL32
2732   {
2733     if(rt1[i]) {
2734       signed char sh,tl,th;
2735       tl=get_reg(i_regs->regmap,rt1[i]);
2736       th=get_reg(i_regs->regmap,rt1[i]|64);
2737       sh=get_reg(i_regs->regmap,rs1[i]|64);
2738       if(tl>=0){
2739         assert(sh>=0);
2740         emit_mov(sh,tl);
2741         if(th>=0) emit_zeroreg(th);
2742         if(imm[i]>32)
2743         {
2744           emit_shrimm(tl,imm[i]&31,tl);
2745         }
2746       }
2747     }
2748   }
2749   if(opcode2[i]==0x3f) // DSRA32
2750   {
2751     if(rt1[i]) {
2752       signed char sh,tl;
2753       tl=get_reg(i_regs->regmap,rt1[i]);
2754       sh=get_reg(i_regs->regmap,rs1[i]|64);
2755       if(tl>=0){
2756         assert(sh>=0);
2757         emit_mov(sh,tl);
2758         if(imm[i]>32)
2759         {
2760           emit_sarimm(tl,imm[i]&31,tl);
2761         }
2762       }
2763     }
2764   }
2765 }
2766
2767 #ifndef shift_assemble
2768 void shift_assemble(int i,struct regstat *i_regs)
2769 {
2770   DebugMessage(M64MSG_ERROR, "Need shift_assemble for this architecture.");
2771   exit(1);
2772 }
2773 #endif
2774
2775 static void load_assemble(int i,struct regstat *i_regs)
2776 {
2777   int s,th,tl,addr,map=-1,cache=-1;
2778   int offset;
2779   int jaddr=0;
2780   int memtarget,c=0;
2781   u_int hr,reglist=0;
2782   th=get_reg(i_regs->regmap,rt1[i]|64);
2783   tl=get_reg(i_regs->regmap,rt1[i]);
2784   s=get_reg(i_regs->regmap,rs1[i]);
2785   offset=imm[i];
2786   for(hr=0;hr<HOST_REGS;hr++) {
2787     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2788   }
2789   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2790   if(s>=0) {
2791     c=(i_regs->wasconst>>s)&1;
2792     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80800000;
2793     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
2794   }
2795   if(tl<0) tl=get_reg(i_regs->regmap,-1);
2796   if(offset||s<0||c) addr=tl;
2797   else addr=s;
2798   //DebugMessage(M64MSG_VERBOSE, "load_assemble: c=%d",c);
2799   //if(c) DebugMessage(M64MSG_VERBOSE, "load_assemble: const=%x",(int)constmap[i][s]+offset);
2800   assert(tl>=0); // Even if the load is a NOP, we must check for pagefaults and I/O
2801   reglist&=~(1<<tl);
2802   if(th>=0) reglist&=~(1<<th);
2803   if(!using_tlb) {
2804     if(!c) {
2805       #ifdef RAM_OFFSET
2806       map=get_reg(i_regs->regmap,ROREG);
2807       if(map<0) emit_loadreg(ROREG,map=HOST_TEMPREG);
2808       #endif
2809 //#define R29_HACK 1
2810       #ifdef R29_HACK
2811       // Strmnnrmn's speed hack
2812       if(rs1[i]!=29||start<0x80001000||start>=0x80800000)
2813       #endif
2814       {
2815         emit_cmpimm(addr,0x800000);
2816         jaddr=(int)out;
2817         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2818         // Hint to branch predictor that the branch is unlikely to be taken
2819         if(rs1[i]>=28)
2820           emit_jno_unlikely(0);
2821         else
2822         #endif
2823         emit_jno(0);
2824       }
2825     }
2826   }else{ // using tlb
2827     int x=0;
2828     if (opcode[i]==0x20||opcode[i]==0x24) x=3; // LB/LBU
2829     if (opcode[i]==0x21||opcode[i]==0x25) x=2; // LH/LHU
2830     map=get_reg(i_regs->regmap,TLREG);
2831     cache=get_reg(i_regs->regmap,MMREG);
2832     assert(map>=0);
2833     reglist&=~(1<<map);
2834     map=do_tlb_r(addr,tl,map,cache,x,-1,-1,c,constmap[i][s]+offset);
2835     do_tlb_r_branch(map,c,constmap[i][s]+offset,&jaddr);
2836   }
2837   int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg
2838   if (opcode[i]==0x20) { // LB
2839     if(!c||memtarget) {
2840       if(!dummy) {
2841         #ifdef HOST_IMM_ADDR32
2842         if(c)
2843           emit_movsbl_tlb((constmap[i][s]+offset)^3,map,tl);
2844         else
2845         #endif
2846         {
2847           //emit_xorimm(addr,3,tl);
2848           //gen_tlb_addr_r(tl,map);
2849           //emit_movsbl_indexed((int)rdram-0x80000000,tl,tl);
2850           int x=0;
2851           if(!c) emit_xorimm(addr,3,tl);
2852           else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
2853           emit_movsbl_indexed_tlb(x,tl,map,tl);
2854         }
2855       }
2856       if(jaddr)
2857         add_stub(LOADB_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2858     }
2859     else
2860       inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2861   }
2862   if (opcode[i]==0x21) { // LH
2863     if(!c||memtarget) {
2864       if(!dummy) {
2865         #ifdef HOST_IMM_ADDR32
2866         if(c)
2867           emit_movswl_tlb((constmap[i][s]+offset)^2,map,tl);
2868         else
2869         #endif
2870         {
2871           int x=0;
2872           if(!c) emit_xorimm(addr,2,tl);
2873           else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
2874           //#ifdef
2875           //emit_movswl_indexed_tlb(x,tl,map,tl);
2876           //else
2877           if(map>=0) {
2878             gen_tlb_addr_r(tl,map);
2879             emit_movswl_indexed(x,tl,tl);
2880           }else{
2881             #ifdef RAM_OFFSET
2882             emit_movswl_indexed(x,tl,tl);
2883             #else
2884             emit_movswl_indexed((int)rdram-0x80000000+x,tl,tl);
2885             #endif
2886           }
2887         }
2888       }
2889       if(jaddr)
2890         add_stub(LOADH_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2891     }
2892     else
2893       inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2894   }
2895   if (opcode[i]==0x23) { // LW
2896     if(!c||memtarget) {
2897       if(!dummy) {
2898         //emit_readword_indexed((int)rdram-0x80000000,addr,tl);
2899         #ifdef HOST_IMM_ADDR32
2900         if(c)
2901           emit_readword_tlb(constmap[i][s]+offset,map,tl);
2902         else
2903         #endif
2904         emit_readword_indexed_tlb(0,addr,map,tl);
2905       }
2906       if(jaddr)
2907         add_stub(LOADW_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2908     }
2909     else
2910       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2911   }
2912   if (opcode[i]==0x24) { // LBU
2913     if(!c||memtarget) {
2914       if(!dummy) {
2915         #ifdef HOST_IMM_ADDR32
2916         if(c)
2917           emit_movzbl_tlb((constmap[i][s]+offset)^3,map,tl);
2918         else
2919         #endif
2920         {
2921           //emit_xorimm(addr,3,tl);
2922           //gen_tlb_addr_r(tl,map);
2923           //emit_movzbl_indexed((int)rdram-0x80000000,tl,tl);
2924           int x=0;
2925           if(!c) emit_xorimm(addr,3,tl);
2926           else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
2927           emit_movzbl_indexed_tlb(x,tl,map,tl);
2928         }
2929       }
2930       if(jaddr)
2931         add_stub(LOADBU_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2932     }
2933     else
2934       inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2935   }
2936   if (opcode[i]==0x25) { // LHU
2937     if(!c||memtarget) {
2938       if(!dummy) {
2939         #ifdef HOST_IMM_ADDR32
2940         if(c)
2941           emit_movzwl_tlb((constmap[i][s]+offset)^2,map,tl);
2942         else
2943         #endif
2944         {
2945           int x=0;
2946           if(!c) emit_xorimm(addr,2,tl);
2947           else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
2948           //#ifdef
2949           //emit_movzwl_indexed_tlb(x,tl,map,tl);
2950           //#else
2951           if(map>=0) {
2952             gen_tlb_addr_r(tl,map);
2953             emit_movzwl_indexed(x,tl,tl);
2954           }else{
2955             #ifdef RAM_OFFSET
2956             emit_movzwl_indexed(x,tl,tl);
2957             #else
2958             emit_movzwl_indexed((int)rdram-0x80000000+x,tl,tl);
2959             #endif
2960           }
2961         }
2962       }
2963       if(jaddr)
2964         add_stub(LOADHU_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2965     }
2966     else
2967       inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2968   }
2969   if (opcode[i]==0x27) { // LWU
2970     assert(th>=0);
2971     if(!c||memtarget) {
2972       if(!dummy) {
2973         //emit_readword_indexed((int)rdram-0x80000000,addr,tl);
2974         #ifdef HOST_IMM_ADDR32
2975         if(c)
2976           emit_readword_tlb(constmap[i][s]+offset,map,tl);
2977         else
2978         #endif
2979         emit_readword_indexed_tlb(0,addr,map,tl);
2980       }
2981       if(jaddr)
2982         add_stub(LOADW_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2983     }
2984     else {
2985       inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2986     }
2987     emit_zeroreg(th);
2988   }
2989   if (opcode[i]==0x37) { // LD
2990     if(!c||memtarget) {
2991       if(!dummy) {
2992         //gen_tlb_addr_r(tl,map);
2993         //if(th>=0) emit_readword_indexed((int)rdram-0x80000000,addr,th);
2994         //emit_readword_indexed((int)rdram-0x7FFFFFFC,addr,tl);
2995         #ifdef HOST_IMM_ADDR32
2996         if(c)
2997           emit_readdword_tlb(constmap[i][s]+offset,map,th,tl);
2998         else
2999         #endif
3000         emit_readdword_indexed_tlb(0,addr,map,th,tl);
3001       }
3002       if(jaddr)
3003         add_stub(LOADD_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
3004     }
3005     else
3006       inline_readstub(LOADD_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
3007   }
3008   //emit_storereg(rt1[i],tl); // DEBUG
3009   //if(opcode[i]==0x23)
3010   //if(opcode[i]==0x24)
3011   //if(opcode[i]==0x23||opcode[i]==0x24)
3012   /*if(opcode[i]==0x21||opcode[i]==0x23||opcode[i]==0x24)
3013   {
3014     //emit_pusha();
3015     save_regs(0x100f);
3016         emit_readword((int)&last_count,ECX);
3017         #if NEW_DYNAREC == NEW_DYNAREC_X86
3018         if(get_reg(i_regs->regmap,CCREG)<0)
3019           emit_loadreg(CCREG,HOST_CCREG);
3020         emit_add(HOST_CCREG,ECX,HOST_CCREG);
3021         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
3022         emit_writeword(HOST_CCREG,(int)&Count);
3023         #endif
3024         #if NEW_DYNAREC == NEW_DYNAREC_ARM
3025         if(get_reg(i_regs->regmap,CCREG)<0)
3026           emit_loadreg(CCREG,0);
3027         else
3028           emit_mov(HOST_CCREG,0);
3029         emit_add(0,ECX,0);
3030         emit_addimm(0,2*ccadj[i],0);
3031         emit_writeword(0,(int)&Count);
3032         #endif
3033     emit_call((int)memdebug);
3034     //emit_popa();
3035     restore_regs(0x100f);
3036   }*/
3037 }
3038
3039 #ifndef loadlr_assemble
3040 static void loadlr_assemble(int i,struct regstat *i_regs)
3041 {
3042   DebugMessage(M64MSG_ERROR, "Need loadlr_assemble for this architecture.");
3043   exit(1);
3044 }
3045 #endif
3046
3047 static void store_assemble(int i,struct regstat *i_regs)
3048 {
3049   int s,th,tl,map=-1,cache=-1;
3050   int addr,temp;
3051   int offset;
3052   int jaddr=0,jaddr2,type;
3053   int memtarget,c=0;
3054   int agr=AGEN1+(i&1);
3055   u_int hr,reglist=0;
3056   th=get_reg(i_regs->regmap,rs2[i]|64);
3057   tl=get_reg(i_regs->regmap,rs2[i]);
3058   s=get_reg(i_regs->regmap,rs1[i]);
3059   temp=get_reg(i_regs->regmap,agr);
3060   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3061   offset=imm[i];
3062   if(s>=0) {
3063     c=(i_regs->wasconst>>s)&1;
3064     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80800000;
3065     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
3066   }
3067   assert(tl>=0);
3068   assert(temp>=0);
3069   for(hr=0;hr<HOST_REGS;hr++) {
3070     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3071   }
3072   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
3073   if(offset||s<0||c) addr=temp;
3074   else addr=s;
3075   if(!using_tlb) {
3076     #ifdef RAM_OFFSET
3077     map=get_reg(i_regs->regmap,ROREG);
3078     if(map<0) emit_loadreg(ROREG,map=HOST_TEMPREG);
3079     #endif
3080     if(!c) {
3081       #ifdef R29_HACK
3082       // Strmnnrmn's speed hack
3083       memtarget=1;
3084       if(rs1[i]!=29||start<0x80001000||start>=0x80800000)
3085       #endif
3086       emit_cmpimm(addr,0x800000);
3087       #ifdef DESTRUCTIVE_SHIFT
3088       if(s==addr) emit_mov(s,temp);
3089       #endif
3090       #ifdef R29_HACK
3091       if(rs1[i]!=29||start<0x80001000||start>=0x80800000)
3092       #endif
3093       {
3094         jaddr=(int)out;
3095         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
3096         // Hint to branch predictor that the branch is unlikely to be taken
3097         if(rs1[i]>=28)
3098           emit_jno_unlikely(0);
3099         else
3100         #endif
3101         emit_jno(0);
3102       }
3103     }
3104   }else{ // using tlb
3105     int x=0;
3106     if (opcode[i]==0x28) x=3; // SB
3107     if (opcode[i]==0x29) x=2; // SH
3108     map=get_reg(i_regs->regmap,TLREG);
3109     cache=get_reg(i_regs->regmap,MMREG);
3110     assert(map>=0);
3111     reglist&=~(1<<map);
3112     map=do_tlb_w(addr,temp,map,cache,x,c,constmap[i][s]+offset);
3113     do_tlb_w_branch(map,c,constmap[i][s]+offset,&jaddr);
3114   }
3115
3116   if (opcode[i]==0x28) { // SB
3117     if(!c||memtarget) {
3118       int x=0;
3119       if(!c) emit_xorimm(addr,3,temp);
3120       else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
3121       //gen_tlb_addr_w(temp,map);
3122       //emit_writebyte_indexed(tl,(int)rdram-0x80000000,temp);
3123       emit_writebyte_indexed_tlb(tl,x,temp,map,temp);
3124     }
3125     type=STOREB_STUB;
3126   }
3127   if (opcode[i]==0x29) { // SH
3128     if(!c||memtarget) {
3129       int x=0;
3130       if(!c) emit_xorimm(addr,2,temp);
3131       else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
3132       //#ifdef
3133       //emit_writehword_indexed_tlb(tl,x,temp,map,temp);
3134       //#else
3135       if(map>=0) {
3136         gen_tlb_addr_w(temp,map);
3137         emit_writehword_indexed(tl,x,temp);
3138       }else
3139         emit_writehword_indexed(tl,(int)rdram-0x80000000+x,temp);
3140     }
3141     type=STOREH_STUB;
3142   }
3143   if (opcode[i]==0x2B) { // SW
3144     if(!c||memtarget)
3145       //emit_writeword_indexed(tl,(int)rdram-0x80000000,addr);
3146       emit_writeword_indexed_tlb(tl,0,addr,map,temp);
3147     type=STOREW_STUB;
3148   }
3149   if (opcode[i]==0x3F) { // SD
3150     if(!c||memtarget) {
3151       if(rs2[i]) {
3152         assert(th>=0);
3153         //emit_writeword_indexed(th,(int)rdram-0x80000000,addr);
3154         //emit_writeword_indexed(tl,(int)rdram-0x7FFFFFFC,addr);
3155         emit_writedword_indexed_tlb(th,tl,0,addr,map,temp);
3156       }else{
3157         // Store zero
3158         //emit_writeword_indexed(tl,(int)rdram-0x80000000,temp);
3159         //emit_writeword_indexed(tl,(int)rdram-0x7FFFFFFC,temp);
3160         emit_writedword_indexed_tlb(tl,tl,0,addr,map,temp);
3161       }
3162     }
3163     type=STORED_STUB;
3164   }
3165   if(!using_tlb) {
3166     if(!c||memtarget) {
3167       #ifdef DESTRUCTIVE_SHIFT
3168       // The x86 shift operation is 'destructive'; it overwrites the
3169       // source register, so we need to make a copy first and use that.
3170       addr=temp;
3171       #endif
3172       #if defined(HOST_IMM8)
3173       int ir=get_reg(i_regs->regmap,INVCP);
3174       assert(ir>=0);
3175       emit_cmpmem_indexedsr12_reg(ir,addr,1);
3176       #else
3177       emit_cmpmem_indexedsr12_imm((int)invalid_code,addr,1);
3178       #endif
3179       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3180       emit_callne(invalidate_addr_reg[addr]);
3181       #else
3182       jaddr2=(int)out;
3183       emit_jne(0);
3184       add_stub(INVCODE_STUB,jaddr2,(int)out,reglist|(1<<HOST_CCREG),addr,0,0,0);
3185       #endif
3186     }
3187   }
3188   if(jaddr) {
3189     add_stub(type,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
3190   } else if(c&&!memtarget) {
3191     inline_writestub(type,i,constmap[i][s]+offset,i_regs->regmap,rs2[i],ccadj[i],reglist);
3192   }
3193   //if(opcode[i]==0x2B || opcode[i]==0x3F)
3194   //if(opcode[i]==0x2B || opcode[i]==0x28)
3195   //if(opcode[i]==0x2B || opcode[i]==0x29)
3196   //if(opcode[i]==0x2B)
3197
3198 // Uncomment for extra debug output:
3199 /*
3200   if(opcode[i]==0x2B || opcode[i]==0x28 || opcode[i]==0x29 || opcode[i]==0x3F)
3201   {
3202     #if NEW_DYNAREC == NEW_DYNAREC_X86
3203     emit_pusha();
3204     #endif
3205     #if NEW_DYNAREC == NEW_DYNAREC_ARM
3206     save_regs(0x100f);
3207     #endif
3208         emit_readword((int)&last_count,ECX);
3209         #if NEW_DYNAREC == NEW_DYNAREC_X86
3210         if(get_reg(i_regs->regmap,CCREG)<0)
3211           emit_loadreg(CCREG,HOST_CCREG);
3212         emit_add(HOST_CCREG,ECX,HOST_CCREG);
3213         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
3214         emit_writeword(HOST_CCREG,(int)&Count);
3215         #endif
3216         #if NEW_DYNAREC == NEW_DYNAREC_ARM
3217         if(get_reg(i_regs->regmap,CCREG)<0)
3218           emit_loadreg(CCREG,0);
3219         else
3220           emit_mov(HOST_CCREG,0);
3221         emit_add(0,ECX,0);
3222         emit_addimm(0,2*ccadj[i],0);
3223         emit_writeword(0,(int)&Count);
3224         #endif
3225     emit_call((int)memdebug);
3226     #if NEW_DYNAREC == NEW_DYNAREC_X86
3227     emit_popa();
3228     #endif
3229     #if NEW_DYNAREC == NEW_DYNAREC_ARM
3230     restore_regs(0x100f);
3231     #endif
3232   }
3233 */
3234 }
3235
3236 static void storelr_assemble(int i,struct regstat *i_regs)
3237 {
3238   int s,th,tl;
3239   int temp;
3240   int temp2;
3241   int offset;
3242   int jaddr=0,jaddr2;
3243   int case1,case2,case3;
3244   int done0,done1,done2;
3245   int memtarget,c=0;
3246   int agr=AGEN1+(i&1);
3247   u_int hr,reglist=0;
3248   th=get_reg(i_regs->regmap,rs2[i]|64);
3249   tl=get_reg(i_regs->regmap,rs2[i]);
3250   s=get_reg(i_regs->regmap,rs1[i]);
3251   temp=get_reg(i_regs->regmap,agr);
3252   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3253   offset=imm[i];
3254   if(s>=0) {
3255     c=(i_regs->isconst>>s)&1;
3256     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80800000;
3257     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
3258   }
3259   assert(tl>=0);
3260   for(hr=0;hr<HOST_REGS;hr++) {
3261     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3262   }
3263   assert(temp>=0);
3264   if(!using_tlb) {
3265     if(!c) {
3266       emit_cmpimm(s<0||offset?temp:s,0x800000);
3267       if(!offset&&s!=temp) emit_mov(s,temp);
3268       jaddr=(int)out;
3269       emit_jno(0);
3270     }
3271     else
3272     {
3273       if(!memtarget||!rs1[i]) {
3274         jaddr=(int)out;
3275         emit_jmp(0);
3276       }
3277     }
3278     #ifdef RAM_OFFSET
3279     int map=get_reg(i_regs->regmap,ROREG);
3280     if(map<0) emit_loadreg(ROREG,map=HOST_TEMPREG);
3281     gen_tlb_addr_w(temp,map);
3282     #else
3283     if((u_int)rdram!=0x80000000)
3284       emit_addimm_no_flags((u_int)rdram-(u_int)0x80000000,temp);
3285     #endif
3286   }else{ // using tlb
3287     int map=get_reg(i_regs->regmap,TLREG);
3288     int cache=get_reg(i_regs->regmap,MMREG);
3289     assert(map>=0);
3290     reglist&=~(1<<map);
3291     map=do_tlb_w(c||s<0||offset?temp:s,temp,map,cache,0,c,constmap[i][s]+offset);
3292     if(!c&&!offset&&s>=0) emit_mov(s,temp);
3293     do_tlb_w_branch(map,c,constmap[i][s]+offset,&jaddr);
3294     if(!jaddr&&!memtarget) {
3295       jaddr=(int)out;
3296       emit_jmp(0);
3297     }
3298     gen_tlb_addr_w(temp,map);
3299   }
3300
3301   if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
3302     temp2=get_reg(i_regs->regmap,FTEMP);
3303     if(!rs2[i]) temp2=th=tl;
3304   }
3305
3306   emit_testimm(temp,2);
3307   case2=(int)out;
3308   emit_jne(0);
3309   emit_testimm(temp,1);
3310   case1=(int)out;
3311   emit_jne(0);
3312   // 0
3313   if (opcode[i]==0x2A) { // SWL
3314     emit_writeword_indexed(tl,0,temp);
3315   }
3316   if (opcode[i]==0x2E) { // SWR
3317     emit_writebyte_indexed(tl,3,temp);
3318   }
3319   if (opcode[i]==0x2C) { // SDL
3320     emit_writeword_indexed(th,0,temp);
3321     if(rs2[i]) emit_mov(tl,temp2);
3322   }
3323   if (opcode[i]==0x2D) { // SDR
3324     emit_writebyte_indexed(tl,3,temp);
3325     if(rs2[i]) emit_shldimm(th,tl,24,temp2);
3326   }
3327   done0=(int)out;
3328   emit_jmp(0);
3329   // 1
3330   set_jump_target(case1,(int)out);
3331   if (opcode[i]==0x2A) { // SWL
3332     // Write 3 msb into three least significant bytes
3333     if(rs2[i]) emit_rorimm(tl,8,tl);
3334     emit_writehword_indexed(tl,-1,temp);
3335     if(rs2[i]) emit_rorimm(tl,16,tl);
3336     emit_writebyte_indexed(tl,1,temp);
3337     if(rs2[i]) emit_rorimm(tl,8,tl);
3338   }
3339   if (opcode[i]==0x2E) { // SWR
3340     // Write two lsb into two most significant bytes
3341     emit_writehword_indexed(tl,1,temp);
3342   }
3343   if (opcode[i]==0x2C) { // SDL
3344     if(rs2[i]) emit_shrdimm(tl,th,8,temp2);
3345     // Write 3 msb into three least significant bytes
3346     if(rs2[i]) emit_rorimm(th,8,th);
3347     emit_writehword_indexed(th,-1,temp);
3348     if(rs2[i]) emit_rorimm(th,16,th);
3349     emit_writebyte_indexed(th,1,temp);
3350     if(rs2[i]) emit_rorimm(th,8,th);
3351   }
3352   if (opcode[i]==0x2D) { // SDR
3353     if(rs2[i]) emit_shldimm(th,tl,16,temp2);
3354     // Write two lsb into two most significant bytes
3355     emit_writehword_indexed(tl,1,temp);
3356   }
3357   done1=(int)out;
3358   emit_jmp(0);
3359   // 2
3360   set_jump_target(case2,(int)out);
3361   emit_testimm(temp,1);
3362   case3=(int)out;
3363   emit_jne(0);
3364   if (opcode[i]==0x2A) { // SWL
3365     // Write two msb into two least significant bytes
3366     if(rs2[i]) emit_rorimm(tl,16,tl);
3367     emit_writehword_indexed(tl,-2,temp);
3368     if(rs2[i]) emit_rorimm(tl,16,tl);
3369   }
3370   if (opcode[i]==0x2E) { // SWR
3371     // Write 3 lsb into three most significant bytes
3372     emit_writebyte_indexed(tl,-1,temp);
3373     if(rs2[i]) emit_rorimm(tl,8,tl);
3374     emit_writehword_indexed(tl,0,temp);
3375     if(rs2[i]) emit_rorimm(tl,24,tl);
3376   }
3377   if (opcode[i]==0x2C) { // SDL
3378     if(rs2[i]) emit_shrdimm(tl,th,16,temp2);
3379     // Write two msb into two least significant bytes
3380     if(rs2[i]) emit_rorimm(th,16,th);
3381     emit_writehword_indexed(th,-2,temp);
3382     if(rs2[i]) emit_rorimm(th,16,th);
3383   }
3384   if (opcode[i]==0x2D) { // SDR
3385     if(rs2[i]) emit_shldimm(th,tl,8,temp2);
3386     // Write 3 lsb into three most significant bytes
3387     emit_writebyte_indexed(tl,-1,temp);
3388     if(rs2[i]) emit_rorimm(tl,8,tl);
3389     emit_writehword_indexed(tl,0,temp);
3390     if(rs2[i]) emit_rorimm(tl,24,tl);
3391   }
3392   done2=(int)out;
3393   emit_jmp(0);
3394   // 3
3395   set_jump_target(case3,(int)out);
3396   if (opcode[i]==0x2A) { // SWL
3397     // Write msb into least significant byte
3398     if(rs2[i]) emit_rorimm(tl,24,tl);
3399     emit_writebyte_indexed(tl,-3,temp);
3400     if(rs2[i]) emit_rorimm(tl,8,tl);
3401   }
3402   if (opcode[i]==0x2E) { // SWR
3403     // Write entire word
3404     emit_writeword_indexed(tl,-3,temp);
3405   }
3406   if (opcode[i]==0x2C) { // SDL
3407     if(rs2[i]) emit_shrdimm(tl,th,24,temp2);
3408     // Write msb into least significant byte
3409     if(rs2[i]) emit_rorimm(th,24,th);
3410     emit_writebyte_indexed(th,-3,temp);
3411     if(rs2[i]) emit_rorimm(th,8,th);
3412   }
3413   if (opcode[i]==0x2D) { // SDR
3414     if(rs2[i]) emit_mov(th,temp2);
3415     // Write entire word
3416     emit_writeword_indexed(tl,-3,temp);
3417   }
3418   set_jump_target(done0,(int)out);
3419   set_jump_target(done1,(int)out);
3420   set_jump_target(done2,(int)out);
3421   if (opcode[i]==0x2C) { // SDL
3422     emit_testimm(temp,4);
3423     done0=(int)out;
3424     emit_jne(0);
3425     emit_andimm(temp,~3,temp);
3426     emit_writeword_indexed(temp2,4,temp);
3427     set_jump_target(done0,(int)out);
3428   }
3429   if (opcode[i]==0x2D) { // SDR
3430     emit_testimm(temp,4);
3431     done0=(int)out;
3432     emit_jeq(0);
3433     emit_andimm(temp,~3,temp);
3434     emit_writeword_indexed(temp2,-4,temp);
3435     set_jump_target(done0,(int)out);
3436   }
3437   if(!c||!memtarget)
3438     add_stub(STORELR_STUB,jaddr,(int)out,0,(int)i_regs,rs2[i],ccadj[i],reglist);
3439   if(!using_tlb) {
3440     #ifdef RAM_OFFSET
3441     int map=get_reg(i_regs->regmap,ROREG);
3442     if(map<0) map=HOST_TEMPREG;
3443     gen_orig_addr_w(temp,map);
3444     #else
3445     emit_addimm_no_flags((u_int)0x80000000-(u_int)rdram,temp);
3446     #endif
3447     #if defined(HOST_IMM8)
3448     int ir=get_reg(i_regs->regmap,INVCP);
3449     assert(ir>=0);
3450     emit_cmpmem_indexedsr12_reg(ir,temp,1);
3451     #else
3452     emit_cmpmem_indexedsr12_imm((int)invalid_code,temp,1);
3453     #endif
3454     #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3455     emit_callne(invalidate_addr_reg[temp]);
3456     #else
3457     jaddr2=(int)out;
3458     emit_jne(0);
3459     add_stub(INVCODE_STUB,jaddr2,(int)out,reglist|(1<<HOST_CCREG),temp,0,0,0);
3460     #endif
3461   }
3462   /*
3463     emit_pusha();
3464     //save_regs(0x100f);
3465         emit_readword((int)&last_count,ECX);
3466         if(get_reg(i_regs->regmap,CCREG)<0)
3467           emit_loadreg(CCREG,HOST_CCREG);
3468         emit_add(HOST_CCREG,ECX,HOST_CCREG);
3469         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
3470         emit_writeword(HOST_CCREG,(int)&Count);
3471     emit_call((int)memdebug);
3472     emit_popa();
3473     //restore_regs(0x100f);
3474   */
3475 }
3476
3477 static void c1ls_assemble(int i,struct regstat *i_regs)
3478 {
3479   int s,th,tl;
3480   int temp,ar;
3481   int map=-1;
3482   int offset;
3483   int c=0;
3484   int jaddr,jaddr2=0,jaddr3,type;
3485   int agr=AGEN1+(i&1);
3486   u_int hr,reglist=0;
3487   th=get_reg(i_regs->regmap,FTEMP|64);
3488   tl=get_reg(i_regs->regmap,FTEMP);
3489   s=get_reg(i_regs->regmap,rs1[i]);
3490   temp=get_reg(i_regs->regmap,agr);
3491   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3492   offset=imm[i];
3493   assert(tl>=0);
3494   assert(rs1[i]>0);
3495   assert(temp>=0);
3496   for(hr=0;hr<HOST_REGS;hr++) {
3497     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3498   }
3499   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
3500   if (opcode[i]==0x31||opcode[i]==0x35) // LWC1/LDC1
3501   {
3502     // Loads use a temporary register which we need to save
3503     reglist|=1<<temp;
3504   }
3505   if (opcode[i]==0x39||opcode[i]==0x3D) // SWC1/SDC1
3506     ar=temp;
3507   else // LWC1/LDC1
3508     ar=tl;
3509   //if(s<0) emit_loadreg(rs1[i],ar); //address_generation does this now
3510   //else c=(i_regs->wasconst>>s)&1;
3511   if(s>=0) c=(i_regs->wasconst>>s)&1;
3512   // Check cop1 unusable
3513   if(!cop1_usable) {
3514     signed char rs=get_reg(i_regs->regmap,CSREG);
3515     assert(rs>=0);
3516     emit_testimm(rs,0x20000000);
3517     jaddr=(int)out;
3518     emit_jeq(0);
3519     add_stub(FP_STUB,jaddr,(int)out,i,rs,(int)i_regs,is_delayslot,0);
3520     cop1_usable=1;
3521   }
3522   if (opcode[i]==0x39) { // SWC1 (get float address)
3523     emit_readword((int)&reg_cop1_simple[(source[i]>>16)&0x1f],tl);
3524   }
3525   if (opcode[i]==0x3D) { // SDC1 (get double address)
3526     emit_readword((int)&reg_cop1_double[(source[i]>>16)&0x1f],tl);
3527   }
3528   // Generate address + offset
3529   if(!using_tlb) {
3530     #ifdef RAM_OFFSET
3531     if (!c||opcode[i]==0x39||opcode[i]==0x3D) // SWC1/SDC1
3532     {
3533       map=get_reg(i_regs->regmap,ROREG);
3534       if(map<0) emit_loadreg(ROREG,map=HOST_TEMPREG);
3535     }
3536     #endif
3537     if(!c)
3538       emit_cmpimm(offset||c||s<0?ar:s,0x800000);
3539   }
3540   else
3541   {
3542     map=get_reg(i_regs->regmap,TLREG);
3543     int cache=get_reg(i_regs->regmap,MMREG);
3544     assert(map>=0);
3545     reglist&=~(1<<map);
3546     if (opcode[i]==0x31||opcode[i]==0x35) { // LWC1/LDC1
3547       map=do_tlb_r(offset||c||s<0?ar:s,ar,map,cache,0,-1,-1,c,constmap[i][s]+offset);
3548     }
3549     if (opcode[i]==0x39||opcode[i]==0x3D) { // SWC1/SDC1
3550       map=do_tlb_w(offset||c||s<0?ar:s,ar,map,cache,0,c,constmap[i][s]+offset);
3551     }
3552   }
3553   if (opcode[i]==0x39) { // SWC1 (read float)
3554     emit_readword_indexed(0,tl,tl);
3555   }
3556   if (opcode[i]==0x3D) { // SDC1 (read double)
3557     emit_readword_indexed(4,tl,th);
3558     emit_readword_indexed(0,tl,tl);
3559   }
3560   if (opcode[i]==0x31) { // LWC1 (get target address)
3561     emit_readword((int)&reg_cop1_simple[(source[i]>>16)&0x1f],temp);
3562   }
3563   if (opcode[i]==0x35) { // LDC1 (get target address)
3564     emit_readword((int)&reg_cop1_double[(source[i]>>16)&0x1f],temp);
3565   }
3566   if(!using_tlb) {
3567     if(!c) {
3568       jaddr2=(int)out;
3569       emit_jno(0);
3570     }
3571     else if(((signed int)(constmap[i][s]+offset))>=(signed int)0x80800000) {
3572       jaddr2=(int)out;
3573       emit_jmp(0); // inline_readstub/inline_writestub?  Very rare case
3574     }
3575     #ifdef DESTRUCTIVE_SHIFT
3576     if (opcode[i]==0x39||opcode[i]==0x3D) { // SWC1/SDC1
3577       if(!offset&&!c&&s>=0) emit_mov(s,ar);
3578     }
3579     #endif
3580   }else{
3581     if (opcode[i]==0x31||opcode[i]==0x35) { // LWC1/LDC1
3582       do_tlb_r_branch(map,c,constmap[i][s]+offset,&jaddr2);
3583     }
3584     if (opcode[i]==0x39||opcode[i]==0x3D) { // SWC1/SDC1
3585       do_tlb_w_branch(map,c,constmap[i][s]+offset,&jaddr2);
3586     }
3587   }
3588   if (opcode[i]==0x31) { // LWC1
3589     //if(s>=0&&!c&&!offset) emit_mov(s,tl);
3590     //gen_tlb_addr_r(ar,map);
3591     //emit_readword_indexed((int)rdram-0x80000000,tl,tl);
3592     #ifdef HOST_IMM_ADDR32
3593     if(c) emit_readword_tlb(constmap[i][s]+offset,map,tl);
3594     else
3595     #endif
3596     emit_readword_indexed_tlb(0,offset||c||s<0?tl:s,map,tl);
3597     type=LOADW_STUB;
3598   }
3599   if (opcode[i]==0x35) { // LDC1
3600     assert(th>=0);
3601     //if(s>=0&&!c&&!offset) emit_mov(s,tl);
3602     //gen_tlb_addr_r(ar,map);
3603     //emit_readword_indexed((int)rdram-0x80000000,tl,th);
3604     //emit_readword_indexed((int)rdram-0x7FFFFFFC,tl,tl);
3605     #ifdef HOST_IMM_ADDR32
3606     if(c) emit_readdword_tlb(constmap[i][s]+offset,map,th,tl);
3607     else
3608     #endif
3609     emit_readdword_indexed_tlb(0,offset||c||s<0?tl:s,map,th,tl);
3610     type=LOADD_STUB;
3611   }
3612   if (opcode[i]==0x39) { // SWC1
3613     //emit_writeword_indexed(tl,(int)rdram-0x80000000,temp);
3614     emit_writeword_indexed_tlb(tl,0,offset||c||s<0?temp:s,map,temp);
3615     type=STOREW_STUB;
3616   }
3617   if (opcode[i]==0x3D) { // SDC1
3618     assert(th>=0);
3619     //emit_writeword_indexed(th,(int)rdram-0x80000000,temp);
3620     //emit_writeword_indexed(tl,(int)rdram-0x7FFFFFFC,temp);
3621     emit_writedword_indexed_tlb(th,tl,0,offset||c||s<0?temp:s,map,temp);
3622     type=STORED_STUB;
3623   }
3624   if(!using_tlb) {
3625     if (opcode[i]==0x39||opcode[i]==0x3D) { // SWC1/SDC1
3626       #ifndef DESTRUCTIVE_SHIFT
3627       temp=offset||c||s<0?ar:s;
3628       #endif
3629       #if defined(HOST_IMM8)
3630       int ir=get_reg(i_regs->regmap,INVCP);
3631       assert(ir>=0);
3632       emit_cmpmem_indexedsr12_reg(ir,temp,1);
3633       #else
3634       emit_cmpmem_indexedsr12_imm((int)invalid_code,temp,1);
3635       #endif
3636       #if defined(HAVE_CONDITIONAL_CALL) && !defined(DESTRUCTIVE_SHIFT)
3637       emit_callne(invalidate_addr_reg[temp]);
3638       #else
3639       jaddr3=(int)out;
3640       emit_jne(0);
3641       add_stub(INVCODE_STUB,jaddr3,(int)out,reglist|(1<<HOST_CCREG),temp,0,0,0);
3642       #endif
3643     }
3644   }
3645   if(jaddr2) add_stub(type,jaddr2,(int)out,i,offset||c||s<0?ar:s,(int)i_regs,ccadj[i],reglist);
3646   if (opcode[i]==0x31) { // LWC1 (write float)
3647     emit_writeword_indexed(tl,0,temp);
3648   }
3649   if (opcode[i]==0x35) { // LDC1 (write double)
3650     emit_writeword_indexed(th,4,temp);
3651     emit_writeword_indexed(tl,0,temp);
3652   }
3653   //if(opcode[i]==0x39)
3654   /*if(opcode[i]==0x39||opcode[i]==0x31)
3655   {
3656     emit_pusha();
3657         emit_readword((int)&last_count,ECX);
3658         if(get_reg(i_regs->regmap,CCREG)<0)
3659           emit_loadreg(CCREG,HOST_CCREG);
3660         emit_add(HOST_CCREG,ECX,HOST_CCREG);
3661         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
3662         emit_writeword(HOST_CCREG,(int)&Count);
3663     emit_call((int)memdebug);
3664     emit_popa();
3665   }*/
3666 }
3667
3668 #ifndef multdiv_assemble
3669 void multdiv_assemble(int i,struct regstat *i_regs)
3670 {
3671   DebugMessage(M64MSG_ERROR, "Need multdiv_assemble for this architecture.");
3672   exit(1);
3673 }
3674 #endif
3675
3676 static void mov_assemble(int i,struct regstat *i_regs)
3677 {
3678   //if(opcode2[i]==0x10||opcode2[i]==0x12) { // MFHI/MFLO
3679   //if(opcode2[i]==0x11||opcode2[i]==0x13) { // MTHI/MTLO
3680   if(rt1[i]) {
3681     signed char sh,sl,th,tl;
3682     th=get_reg(i_regs->regmap,rt1[i]|64);
3683     tl=get_reg(i_regs->regmap,rt1[i]);
3684     //assert(tl>=0);
3685     if(tl>=0) {
3686       sh=get_reg(i_regs->regmap,rs1[i]|64);
3687       sl=get_reg(i_regs->regmap,rs1[i]);
3688       if(sl>=0) emit_mov(sl,tl);
3689       else emit_loadreg(rs1[i],tl);
3690       if(th>=0) {
3691         if(sh>=0) emit_mov(sh,th);
3692         else emit_loadreg(rs1[i]|64,th);
3693       }
3694     }
3695   }
3696 }
3697
3698 #ifndef fconv_assemble
3699 void fconv_assemble(int i,struct regstat *i_regs)
3700 {
3701   DebugMessage(M64MSG_ERROR, "Need fconv_assemble for this architecture.");
3702   exit(1);
3703 }
3704 #endif
3705
3706 #if 0
3707 static void float_assemble(int i,struct regstat *i_regs)
3708 {
3709   DebugMessage(M64MSG_ERROR, "Need float_assemble for this architecture.");
3710   exit(1);
3711 }
3712 #endif
3713
3714 static void syscall_assemble(int i,struct regstat *i_regs)
3715 {
3716   signed char ccreg=get_reg(i_regs->regmap,CCREG);
3717   assert(ccreg==HOST_CCREG);
3718   assert(!is_delayslot);
3719   emit_movimm(start+i*4,EAX); // Get PC
3720   emit_addimm(HOST_CCREG,CLOCK_DIVIDER*ccadj[i],HOST_CCREG); // CHECK: is this right?  There should probably be an extra cycle...
3721   emit_jmp((int)jump_syscall);
3722 }
3723
3724 static void ds_assemble(int i,struct regstat *i_regs)
3725 {
3726   is_delayslot=1;
3727   switch(itype[i]) {
3728     case ALU:
3729       alu_assemble(i,i_regs);break;
3730     case IMM16:
3731       imm16_assemble(i,i_regs);break;
3732     case SHIFT:
3733       shift_assemble(i,i_regs);break;
3734     case SHIFTIMM:
3735       shiftimm_assemble(i,i_regs);break;
3736     case LOAD:
3737       load_assemble(i,i_regs);break;
3738     case LOADLR:
3739       loadlr_assemble(i,i_regs);break;
3740     case STORE:
3741       store_assemble(i,i_regs);break;
3742     case STORELR:
3743       storelr_assemble(i,i_regs);break;
3744     case COP0:
3745       cop0_assemble(i,i_regs);break;
3746     case COP1:
3747       cop1_assemble(i,i_regs);break;
3748     case C1LS:
3749       c1ls_assemble(i,i_regs);break;
3750     case FCONV:
3751       fconv_assemble(i,i_regs);break;
3752     case FLOAT:
3753       float_assemble(i,i_regs);break;
3754     case FCOMP:
3755       fcomp_assemble(i,i_regs);break;
3756     case MULTDIV:
3757       multdiv_assemble(i,i_regs);break;
3758     case MOV:
3759       mov_assemble(i,i_regs);break;
3760     case SYSCALL:
3761     case SPAN:
3762     case UJUMP:
3763     case RJUMP:
3764     case CJUMP:
3765     case SJUMP:
3766     case FJUMP:
3767       DebugMessage(M64MSG_VERBOSE, "Jump in the delay slot.  This is probably a bug.");
3768   }
3769   is_delayslot=0;
3770 }
3771
3772 // Is the branch target a valid internal jump?
3773 static int internal_branch(uint64_t i_is32,int addr)
3774 {
3775   if(addr&1) return 0; // Indirect (register) jump
3776   if(addr>=start && addr<start+slen*4-4)
3777   {
3778     int t=(addr-start)>>2;
3779     // Delay slots are not valid branch targets
3780     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP||itype[t-1]==FJUMP)) return 0;
3781     // 64 -> 32 bit transition requires a recompile
3782     /*if(is32[t]&~unneeded_reg_upper[t]&~i_is32)
3783     {
3784       if(requires_32bit[t]&~i_is32) DebugMessage(M64MSG_VERBOSE, "optimizable: no");
3785       else DebugMessage(M64MSG_VERBOSE, "optimizable: yes");
3786     }*/
3787     //if(is32[t]&~unneeded_reg_upper[t]&~i_is32) return 0;
3788     if(requires_32bit[t]&~i_is32) return 0;
3789     else return 1;
3790   }
3791   return 0;
3792 }
3793
3794 #ifndef wb_invalidate
3795 static void wb_invalidate(signed char pre[],signed char entry[],uint64_t dirty,uint64_t is32,
3796   uint64_t u,uint64_t uu)
3797 {
3798   int hr;
3799   for(hr=0;hr<HOST_REGS;hr++) {
3800     if(hr!=EXCLUDE_REG) {
3801       if(pre[hr]!=entry[hr]) {
3802         if(pre[hr]>=0) {
3803           if((dirty>>hr)&1) {
3804             if(get_reg(entry,pre[hr])<0) {
3805               if(pre[hr]<64) {
3806                 if(!((u>>pre[hr])&1)) {
3807                   emit_storereg(pre[hr],hr);
3808                   if( ((is32>>pre[hr])&1) && !((uu>>pre[hr])&1) ) {
3809                     emit_sarimm(hr,31,hr);
3810                     emit_storereg(pre[hr]|64,hr);
3811                   }
3812                 }
3813               }else{
3814                 if(!((uu>>(pre[hr]&63))&1) && !((is32>>(pre[hr]&63))&1)) {
3815                   emit_storereg(pre[hr],hr);
3816                 }
3817               }
3818             }
3819           }
3820         }
3821       }
3822     }
3823   }
3824   // Move from one register to another (no writeback)
3825   for(hr=0;hr<HOST_REGS;hr++) {
3826     if(hr!=EXCLUDE_REG) {
3827       if(pre[hr]!=entry[hr]) {
3828         if(pre[hr]>=0&&(pre[hr]&63)<TEMPREG) {
3829           int nr;
3830           if((nr=get_reg(entry,pre[hr]))>=0) {
3831             emit_mov(hr,nr);
3832           }
3833         }
3834       }
3835     }
3836   }
3837 }
3838 #endif
3839
3840 // Load the specified registers
3841 // This only loads the registers given as arguments because
3842 // we don't want to load things that will be overwritten
3843 static void load_regs(signed char entry[],signed char regmap[],int is32,int rs1,int rs2)
3844 {
3845   int hr;
3846   // Load 32-bit regs
3847   for(hr=0;hr<HOST_REGS;hr++) {
3848     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3849       if(entry[hr]!=regmap[hr]) {
3850         if(regmap[hr]==rs1||regmap[hr]==rs2)
3851         {
3852           if(regmap[hr]==0) {
3853             emit_zeroreg(hr);
3854           }
3855           else
3856           {
3857             emit_loadreg(regmap[hr],hr);
3858           }
3859         }
3860       }
3861     }
3862   }
3863   //Load 64-bit regs
3864   for(hr=0;hr<HOST_REGS;hr++) {
3865     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
3866       if(entry[hr]!=regmap[hr]) {
3867         if(regmap[hr]-64==rs1||regmap[hr]-64==rs2)
3868         {
3869           assert(regmap[hr]!=64);
3870           if((is32>>(regmap[hr]&63))&1) {
3871             int lr=get_reg(regmap,regmap[hr]-64);
3872             if(lr>=0)
3873               emit_sarimm(lr,31,hr);
3874             else
3875               emit_loadreg(regmap[hr],hr);
3876           }
3877           else
3878           {
3879             emit_loadreg(regmap[hr],hr);
3880           }
3881         }
3882       }
3883     }
3884   }
3885 }
3886
3887 // Load registers prior to the start of a loop
3888 // so that they are not loaded within the loop
3889 static void loop_preload(signed char pre[],signed char entry[])
3890 {
3891   int hr;
3892   for(hr=0;hr<HOST_REGS;hr++) {
3893     if(hr!=EXCLUDE_REG) {
3894       if(pre[hr]!=entry[hr]) {
3895         if(entry[hr]>=0) {
3896           if(get_reg(pre,entry[hr])<0) {
3897             assem_debug("loop preload:");
3898             //DebugMessage(M64MSG_VERBOSE, "loop preload: %d",hr);
3899             if(entry[hr]==0) {
3900               emit_zeroreg(hr);
3901             }
3902             else if(entry[hr]<TEMPREG)
3903             {
3904               emit_loadreg(entry[hr],hr);
3905             }
3906             else if(entry[hr]-64<TEMPREG)
3907             {
3908               emit_loadreg(entry[hr],hr);
3909             }
3910           }
3911         }
3912       }
3913     }
3914   }
3915 }
3916
3917 // Generate address for load/store instruction
3918 static void address_generation(int i,struct regstat *i_regs,signed char entry[])
3919 {
3920   if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS) {
3921     int ra;
3922     int agr=AGEN1+(i&1);
3923     int mgr=MGEN1+(i&1);
3924     if(itype[i]==LOAD) {
3925       ra=get_reg(i_regs->regmap,rt1[i]);
3926       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3927       assert(ra>=0);
3928     }
3929     if(itype[i]==LOADLR) {
3930       ra=get_reg(i_regs->regmap,FTEMP);
3931     }
3932     if(itype[i]==STORE||itype[i]==STORELR) {
3933       ra=get_reg(i_regs->regmap,agr);
3934       if(ra<0) ra=get_reg(i_regs->regmap,-1);
3935     }
3936     if(itype[i]==C1LS) {
3937       if (opcode[i]==0x31||opcode[i]==0x35) // LWC1/LDC1
3938         ra=get_reg(i_regs->regmap,FTEMP);
3939       else { // SWC1/SDC1
3940         ra=get_reg(i_regs->regmap,agr);
3941         if(ra<0) ra=get_reg(i_regs->regmap,-1);
3942       }
3943     }
3944     int rs=get_reg(i_regs->regmap,rs1[i]);
3945     int rm=get_reg(i_regs->regmap,TLREG);
3946     if(ra>=0) {
3947       int offset=imm[i];
3948       int c=(i_regs->wasconst>>rs)&1;
3949       if(rs1[i]==0) {
3950         // Using r0 as a base address
3951         /*if(rm>=0) {
3952           if(!entry||entry[rm]!=mgr) {
3953             generate_map_const(offset,rm);
3954           } // else did it in the previous cycle
3955         }*/
3956         if(!entry||entry[ra]!=agr) {
3957           if (opcode[i]==0x22||opcode[i]==0x26) {
3958             emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
3959           }else if (opcode[i]==0x1a||opcode[i]==0x1b) {
3960             emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
3961           }else{
3962             emit_movimm(offset,ra);
3963           }
3964         } // else did it in the previous cycle
3965       }
3966       else if(rs<0) {
3967         if(!entry||entry[ra]!=rs1[i])
3968           emit_loadreg(rs1[i],ra);
3969         //if(!entry||entry[ra]!=rs1[i])
3970         //  DebugMessage(M64MSG_VERBOSE, "poor load scheduling!");
3971       }
3972       else if(c) {
3973         if(rm>=0) {
3974           if(!entry||entry[rm]!=mgr) {
3975             if(itype[i]==STORE||itype[i]==STORELR||opcode[i]==0x39||opcode[i]==0x3D) {
3976               // Stores to memory go thru the mapper to detect self-modifying
3977               // code, loads don't.
3978               if((unsigned int)(constmap[i][rs]+offset)>=0xC0000000 ||
3979                  (unsigned int)(constmap[i][rs]+offset)<0x80800000 )
3980                 generate_map_const(constmap[i][rs]+offset,rm);
3981             }else{
3982               if((signed int)(constmap[i][rs]+offset)>=(signed int)0xC0000000)
3983                 generate_map_const(constmap[i][rs]+offset,rm);
3984             }
3985           }
3986         }
3987         if(rs1[i]!=rt1[i]||itype[i]!=LOAD) {
3988           if(!entry||entry[ra]!=agr) {
3989             if (opcode[i]==0x22||opcode[i]==0x26) { // LWL/LWR
3990               #ifdef RAM_OFFSET
3991               if((signed int)constmap[i][rs]+offset<(signed int)0x80800000)
3992                 emit_movimm(((constmap[i][rs]+offset)&0xFFFFFFFC)+(int)rdram-0x80000000,ra);
3993               else
3994               #endif
3995               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFFC,ra);
3996             }else if (opcode[i]==0x1a||opcode[i]==0x1b) { // LDL/LDR
3997               #ifdef RAM_OFFSET
3998               if((signed int)constmap[i][rs]+offset<(signed int)0x80800000)
3999                 emit_movimm(((constmap[i][rs]+offset)&0xFFFFFFF8)+(int)rdram-0x80000000,ra);
4000               else
4001               #endif
4002               emit_movimm((constmap[i][rs]+offset)&0xFFFFFFF8,ra);
4003             }else{
4004               #ifdef HOST_IMM_ADDR32
4005               if((itype[i]!=LOAD&&opcode[i]!=0x31&&opcode[i]!=0x35) ||
4006                  (using_tlb&&((signed int)constmap[i][rs]+offset)>=(signed int)0xC0000000))
4007               #endif
4008               #ifdef RAM_OFFSET
4009               if((itype[i]==LOAD||opcode[i]==0x31||opcode[i]==0x35)&&(signed int)constmap[i][rs]+offset<(signed int)0x80800000)
4010                 emit_movimm(constmap[i][rs]+offset+(int)rdram-0x80000000,ra);
4011               else
4012               #endif
4013               emit_movimm(constmap[i][rs]+offset,ra);
4014             }
4015           } // else did it in the previous cycle
4016         } // else load_consts already did it
4017       }
4018       if(offset&&!c&&rs1[i]) {
4019         if(rs>=0) {
4020           emit_addimm(rs,offset,ra);
4021         }else{
4022           emit_addimm(ra,offset,ra);
4023         }
4024       }
4025     }
4026   }
4027   // Preload constants for next instruction
4028   if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS) {
4029     int agr,ra;
4030     #ifndef HOST_IMM_ADDR32
4031     // Mapper entry
4032     agr=MGEN1+((i+1)&1);
4033     ra=get_reg(i_regs->regmap,agr);
4034     if(ra>=0) {
4035       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
4036       int offset=imm[i+1];
4037       int c=(regs[i+1].wasconst>>rs)&1;
4038       if(c) {
4039         if(itype[i+1]==STORE||itype[i+1]==STORELR||opcode[i+1]==0x39||opcode[i+1]==0x3D) {
4040           // Stores to memory go thru the mapper to detect self-modifying
4041           // code, loads don't.
4042           if((unsigned int)(constmap[i+1][rs]+offset)>=0xC0000000 ||
4043              (unsigned int)(constmap[i+1][rs]+offset)<0x80800000 )
4044             generate_map_const(constmap[i+1][rs]+offset,ra);
4045         }else{
4046           if((signed int)(constmap[i+1][rs]+offset)>=(signed int)0xC0000000)
4047             generate_map_const(constmap[i+1][rs]+offset,ra);
4048         }
4049       }
4050       /*else if(rs1[i]==0) {
4051         generate_map_const(offset,ra);
4052       }*/
4053     }
4054     #endif
4055     // Actual address
4056     agr=AGEN1+((i+1)&1);
4057     ra=get_reg(i_regs->regmap,agr);
4058     if(ra>=0) {
4059       int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
4060       int offset=imm[i+1];
4061       int c=(regs[i+1].wasconst>>rs)&1;
4062       if(c&&(rs1[i+1]!=rt1[i+1]||itype[i+1]!=LOAD)) {
4063         if (opcode[i+1]==0x22||opcode[i+1]==0x26) { // LWL/LWR
4064           #ifdef RAM_OFFSET
4065           if((signed int)constmap[i+1][rs]+offset<(signed int)0x80800000)
4066             emit_movimm(((constmap[i+1][rs]+offset)&0xFFFFFFFC)+(int)rdram-0x80000000,ra);
4067           else
4068           #endif
4069           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFFC,ra);
4070         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) { // LDL/LDR
4071           #ifdef RAM_OFFSET
4072           if((signed int)constmap[i+1][rs]+offset<(signed int)0x80800000)
4073             emit_movimm(((constmap[i+1][rs]+offset)&0xFFFFFFF8)+(int)rdram-0x80000000,ra);
4074           else
4075           #endif
4076           emit_movimm((constmap[i+1][rs]+offset)&0xFFFFFFF8,ra);
4077         }else{
4078           #ifdef HOST_IMM_ADDR32
4079           if((itype[i+1]!=LOAD&&opcode[i+1]!=0x31&&opcode[i+1]!=0x35) ||
4080              (using_tlb&&((signed int)constmap[i+1][rs]+offset)>=(signed int)0xC0000000))
4081           #endif
4082           #ifdef RAM_OFFSET
4083           if((itype[i+1]==LOAD||opcode[i+1]==0x31||opcode[i+1]==0x35)&&(signed int)constmap[i+1][rs]+offset<(signed int)0x80800000)
4084             emit_movimm(constmap[i+1][rs]+offset+(int)rdram-0x80000000,ra);
4085           else
4086           #endif
4087           emit_movimm(constmap[i+1][rs]+offset,ra);
4088         }
4089       }
4090       else if(rs1[i+1]==0) {
4091         // Using r0 as a base address
4092         if (opcode[i+1]==0x22||opcode[i+1]==0x26) {
4093           emit_movimm(offset&0xFFFFFFFC,ra); // LWL/LWR
4094         }else if (opcode[i+1]==0x1a||opcode[i+1]==0x1b) {
4095           emit_movimm(offset&0xFFFFFFF8,ra); // LDL/LDR
4096         }else{
4097           emit_movimm(offset,ra);
4098         }
4099       }
4100     }
4101   }
4102 }
4103
4104 static int get_final_value(int hr, int i, int *value)
4105 {
4106   int reg=regs[i].regmap[hr];
4107   while(i<slen-1) {
4108     if(regs[i+1].regmap[hr]!=reg) break;
4109     if(!((regs[i+1].isconst>>hr)&1)) break;
4110     if(bt[i+1]) break;
4111     i++;
4112   }
4113   if(i<slen-1) {
4114     if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP) {
4115       *value=constmap[i][hr];
4116       return 1;
4117     }
4118     if(!bt[i+1]) {
4119       if(itype[i+1]==UJUMP||itype[i+1]==RJUMP||itype[i+1]==CJUMP||itype[i+1]==SJUMP) {
4120         // Load in delay slot, out-of-order execution
4121         if(itype[i+2]==LOAD&&rs1[i+2]==reg&&rt1[i+2]==reg&&((regs[i+1].wasconst>>hr)&1))
4122         {
4123           #ifdef HOST_IMM_ADDR32
4124           if(!using_tlb||((signed int)constmap[i][hr]+imm[i+2])<(signed int)0xC0000000) return 0;
4125           #endif
4126           #ifdef RAM_OFFSET
4127           if((signed int)constmap[i][hr]+imm[i+2]<(signed int)0x80800000)
4128             *value=constmap[i][hr]+imm[i+2]+(int)rdram-0x80000000;
4129           else
4130           #endif
4131           // Precompute load address
4132           *value=constmap[i][hr]+imm[i+2];
4133           return 1;
4134         }
4135       }
4136       if(itype[i+1]==LOAD&&rs1[i+1]==reg&&rt1[i+1]==reg)
4137       {
4138         #ifdef HOST_IMM_ADDR32
4139         if(!using_tlb||((signed int)constmap[i][hr]+imm[i+1])<(signed int)0xC0000000) return 0;
4140         #endif
4141         #ifdef RAM_OFFSET
4142         if((signed int)constmap[i][hr]+imm[i+1]<(signed int)0x80800000)
4143           *value=constmap[i][hr]+imm[i+1]+(int)rdram-0x80000000;
4144         else
4145         #endif
4146         // Precompute load address
4147         *value=constmap[i][hr]+imm[i+1];
4148         //DebugMessage(M64MSG_VERBOSE, "c=%x imm=%x",(int)constmap[i][hr],imm[i+1]);
4149         return 1;
4150       }
4151     }
4152   }
4153   *value=constmap[i][hr];
4154   //DebugMessage(M64MSG_VERBOSE, "c=%x",(int)constmap[i][hr]);
4155   if(i==slen-1) return 1;
4156   if(reg<64) {
4157     return !((unneeded_reg[i+1]>>reg)&1);
4158   }else{
4159     return !((unneeded_reg_upper[i+1]>>reg)&1);
4160   }
4161 }
4162
4163 // Load registers with known constants
4164 static void load_consts(signed char pre[],signed char regmap[],int is32,int i)
4165 {
4166   int hr;
4167   // Load 32-bit regs
4168   for(hr=0;hr<HOST_REGS;hr++) {
4169     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
4170       //if(entry[hr]!=regmap[hr]) {
4171       if(i==0||!((regs[i-1].isconst>>hr)&1)||pre[hr]!=regmap[hr]||bt[i]) {
4172         if(((regs[i].isconst>>hr)&1)&&regmap[hr]<64&&regmap[hr]>0) {
4173           int value;
4174           if(get_final_value(hr,i,&value)) {
4175             if(value==0) {
4176               emit_zeroreg(hr);
4177             }
4178             else {
4179               emit_movimm(value,hr);
4180             }
4181           }
4182         }
4183       }
4184     }
4185   }
4186   // Load 64-bit regs
4187   for(hr=0;hr<HOST_REGS;hr++) {
4188     if(hr!=EXCLUDE_REG&&regmap[hr]>=0) {
4189       //if(entry[hr]!=regmap[hr]) {
4190       if(i==0||!((regs[i-1].isconst>>hr)&1)||pre[hr]!=regmap[hr]||bt[i]) {
4191         if(((regs[i].isconst>>hr)&1)&&regmap[hr]>64) {
4192           if((is32>>(regmap[hr]&63))&1) {
4193             int lr=get_reg(regmap,regmap[hr]-64);
4194             assert(lr>=0);
4195             emit_sarimm(lr,31,hr);
4196           }
4197           else
4198           {
4199             int value;
4200             if(get_final_value(hr,i,&value)) {
4201               if(value==0) {
4202                 emit_zeroreg(hr);
4203               }
4204               else {
4205                 emit_movimm(value,hr);
4206               }
4207             }
4208           }
4209         }
4210       }
4211     }
4212   }
4213 }
4214 static void load_all_consts(signed char regmap[],int is32,u_int dirty,int i)
4215 {
4216   int hr;
4217   // Load 32-bit regs
4218   for(hr=0;hr<HOST_REGS;hr++) {
4219     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
4220       if(((regs[i].isconst>>hr)&1)&&regmap[hr]<64&&regmap[hr]>0) {
4221         int value=constmap[i][hr];
4222         if(value==0) {
4223           emit_zeroreg(hr);
4224         }
4225         else {
4226           emit_movimm(value,hr);
4227         }
4228       }
4229     }
4230   }
4231   // Load 64-bit regs
4232   for(hr=0;hr<HOST_REGS;hr++) {
4233     if(hr!=EXCLUDE_REG&&regmap[hr]>=0&&((dirty>>hr)&1)) {
4234       if(((regs[i].isconst>>hr)&1)&&regmap[hr]>64) {
4235         if((is32>>(regmap[hr]&63))&1) {
4236           int lr=get_reg(regmap,regmap[hr]-64);
4237           assert(lr>=0);
4238           emit_sarimm(lr,31,hr);
4239         }
4240         else
4241         {
4242           int value=constmap[i][hr];
4243           if(value==0) {
4244             emit_zeroreg(hr);
4245           }
4246           else {
4247             emit_movimm(value,hr);
4248           }
4249         }
4250       }
4251     }
4252   }
4253 }
4254
4255 // Write out all dirty registers (except cycle count)
4256 static void wb_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty)
4257 {
4258   int hr;
4259   for(hr=0;hr<HOST_REGS;hr++) {
4260     if(hr!=EXCLUDE_REG) {
4261       if(i_regmap[hr]>0) {
4262         if(i_regmap[hr]!=CCREG) {
4263           if((i_dirty>>hr)&1) {
4264             if(i_regmap[hr]<64) {
4265               emit_storereg(i_regmap[hr],hr);
4266               if( ((i_is32>>i_regmap[hr])&1) ) {
4267                 #ifdef DESTRUCTIVE_WRITEBACK
4268                 emit_sarimm(hr,31,hr);
4269                 emit_storereg(i_regmap[hr]|64,hr);
4270                 #else
4271                 emit_sarimm(hr,31,HOST_TEMPREG);
4272                 emit_storereg(i_regmap[hr]|64,HOST_TEMPREG);
4273                 #endif
4274               }
4275             }else{
4276               if( !((i_is32>>(i_regmap[hr]&63))&1) ) {
4277                 emit_storereg(i_regmap[hr],hr);
4278               }
4279             }
4280           }
4281         }
4282       }
4283     }
4284   }
4285 }
4286 // Write out dirty registers that we need to reload (pair with load_needed_regs)
4287 // This writes the registers not written by store_regs_bt
4288 static void wb_needed_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr)
4289 {
4290   int hr;
4291   int t=(addr-start)>>2;
4292   for(hr=0;hr<HOST_REGS;hr++) {
4293     if(hr!=EXCLUDE_REG) {
4294       if(i_regmap[hr]>0) {
4295         if(i_regmap[hr]!=CCREG) {
4296           if(i_regmap[hr]==regs[t].regmap_entry[hr] && ((regs[t].dirty>>hr)&1) && !(((i_is32&~regs[t].was32&~unneeded_reg_upper[t])>>(i_regmap[hr]&63))&1)) {
4297             if((i_dirty>>hr)&1) {
4298               if(i_regmap[hr]<64) {
4299                 emit_storereg(i_regmap[hr],hr);
4300                 if( ((i_is32>>i_regmap[hr])&1) ) {
4301                   #ifdef DESTRUCTIVE_WRITEBACK
4302                   emit_sarimm(hr,31,hr);
4303                   emit_storereg(i_regmap[hr]|64,hr);
4304                   #else
4305                   emit_sarimm(hr,31,HOST_TEMPREG);
4306                   emit_storereg(i_regmap[hr]|64,HOST_TEMPREG);
4307                   #endif
4308                 }
4309               }else{
4310                 if( !((i_is32>>(i_regmap[hr]&63))&1) ) {
4311                   emit_storereg(i_regmap[hr],hr);
4312                 }
4313               }
4314             }
4315           }
4316         }
4317       }
4318     }
4319   }
4320 }
4321
4322 // Load all registers (except cycle count)
4323 static void load_all_regs(signed char i_regmap[])
4324 {
4325   int hr;
4326   for(hr=0;hr<HOST_REGS;hr++) {
4327     if(hr!=EXCLUDE_REG) {
4328       if(i_regmap[hr]==0) {
4329         emit_zeroreg(hr);
4330       }
4331       else
4332       if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4333       {
4334         emit_loadreg(i_regmap[hr],hr);
4335       }
4336     }
4337   }
4338 }
4339
4340 // Load all current registers also needed by next instruction
4341 static void load_needed_regs(signed char i_regmap[],signed char next_regmap[])
4342 {
4343   int hr;
4344   for(hr=0;hr<HOST_REGS;hr++) {
4345     if(hr!=EXCLUDE_REG) {
4346       if(get_reg(next_regmap,i_regmap[hr])>=0) {
4347         if(i_regmap[hr]==0) {
4348           emit_zeroreg(hr);
4349         }
4350         else
4351         if(i_regmap[hr]>0 && (i_regmap[hr]&63)<TEMPREG && i_regmap[hr]!=CCREG)
4352         {
4353           emit_loadreg(i_regmap[hr],hr);
4354         }
4355       }
4356     }
4357   }
4358 }
4359
4360 // Load all regs, storing cycle count if necessary
4361 static void load_regs_entry(int t)
4362 {
4363   int hr;
4364   if(is_ds[t]) emit_addimm(HOST_CCREG,CLOCK_DIVIDER,HOST_CCREG);
4365   else if(ccadj[t]) emit_addimm(HOST_CCREG,-ccadj[t]*CLOCK_DIVIDER,HOST_CCREG);
4366   if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4367     emit_storereg(CCREG,HOST_CCREG);
4368   }
4369   // Load 32-bit regs
4370   for(hr=0;hr<HOST_REGS;hr++) {
4371     if(regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4372       if(regs[t].regmap_entry[hr]==0) {
4373         emit_zeroreg(hr);
4374       }
4375       else if(regs[t].regmap_entry[hr]!=CCREG)
4376       {
4377         emit_loadreg(regs[t].regmap_entry[hr],hr);
4378       }
4379     }
4380   }
4381   // Load 64-bit regs
4382   for(hr=0;hr<HOST_REGS;hr++) {
4383     if(regs[t].regmap_entry[hr]>=64&&regs[t].regmap_entry[hr]<TEMPREG+64) {
4384       assert(regs[t].regmap_entry[hr]!=64);
4385       if((regs[t].was32>>(regs[t].regmap_entry[hr]&63))&1) {
4386         int lr=get_reg(regs[t].regmap_entry,regs[t].regmap_entry[hr]-64);
4387         if(lr<0) {
4388           emit_loadreg(regs[t].regmap_entry[hr],hr);
4389         }
4390         else
4391         {
4392           emit_sarimm(lr,31,hr);
4393         }
4394       }
4395       else
4396       {
4397         emit_loadreg(regs[t].regmap_entry[hr],hr);
4398       }
4399     }
4400   }
4401 }
4402
4403 // Store dirty registers prior to branch
4404 static void store_regs_bt(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr)
4405 {
4406   if(internal_branch(i_is32,addr))
4407   {
4408     int t=(addr-start)>>2;
4409     int hr;
4410     for(hr=0;hr<HOST_REGS;hr++) {
4411       if(hr!=EXCLUDE_REG) {
4412         if(i_regmap[hr]>0 && i_regmap[hr]!=CCREG) {
4413           if(i_regmap[hr]!=regs[t].regmap_entry[hr] || !((regs[t].dirty>>hr)&1) || (((i_is32&~regs[t].was32&~unneeded_reg_upper[t])>>(i_regmap[hr]&63))&1)) {
4414             if((i_dirty>>hr)&1) {
4415               if(i_regmap[hr]<64) {
4416                 if(!((unneeded_reg[t]>>i_regmap[hr])&1)) {
4417                   emit_storereg(i_regmap[hr],hr);
4418                   if( ((i_is32>>i_regmap[hr])&1) && !((unneeded_reg_upper[t]>>i_regmap[hr])&1) ) {
4419                     #ifdef DESTRUCTIVE_WRITEBACK
4420                     emit_sarimm(hr,31,hr);
4421                     emit_storereg(i_regmap[hr]|64,hr);
4422                     #else
4423                     emit_sarimm(hr,31,HOST_TEMPREG);
4424                     emit_storereg(i_regmap[hr]|64,HOST_TEMPREG);
4425                     #endif
4426                   }
4427                 }
4428               }else{
4429                 if( !((i_is32>>(i_regmap[hr]&63))&1) && !((unneeded_reg_upper[t]>>(i_regmap[hr]&63))&1) ) {
4430                   emit_storereg(i_regmap[hr],hr);
4431                 }
4432               }
4433             }
4434           }
4435         }
4436       }
4437     }
4438   }
4439   else
4440   {
4441     // Branch out of this block, write out all dirty regs
4442     wb_dirtys(i_regmap,i_is32,i_dirty);
4443   }
4444 }
4445
4446 // Load all needed registers for branch target
4447 static void load_regs_bt(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr)
4448 {
4449   //if(addr>=start && addr<(start+slen*4))
4450   if(internal_branch(i_is32,addr))
4451   {
4452     int t=(addr-start)>>2;
4453     int hr;
4454     // Store the cycle count before loading something else
4455     if(i_regmap[HOST_CCREG]!=CCREG) {
4456       assert(i_regmap[HOST_CCREG]==-1);
4457     }
4458     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) {
4459       emit_storereg(CCREG,HOST_CCREG);
4460     }
4461     // Load 32-bit regs
4462     for(hr=0;hr<HOST_REGS;hr++) {
4463       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=0&&regs[t].regmap_entry[hr]<TEMPREG) {
4464         #ifdef DESTRUCTIVE_WRITEBACK
4465         if(i_regmap[hr]!=regs[t].regmap_entry[hr] || ( !((regs[t].dirty>>hr)&1) && ((i_dirty>>hr)&1) && (((i_is32&~unneeded_reg_upper[t])>>i_regmap[hr])&1) ) || (((i_is32&~regs[t].was32&~unneeded_reg_upper[t])>>(i_regmap[hr]&63))&1)) {
4466         #else
4467         if(i_regmap[hr]!=regs[t].regmap_entry[hr] ) {
4468         #endif
4469           if(regs[t].regmap_entry[hr]==0) {
4470             emit_zeroreg(hr);
4471           }
4472           else if(regs[t].regmap_entry[hr]!=CCREG)
4473           {
4474             emit_loadreg(regs[t].regmap_entry[hr],hr);
4475           }
4476         }
4477       }
4478     }
4479     //Load 64-bit regs
4480     for(hr=0;hr<HOST_REGS;hr++) {
4481       if(hr!=EXCLUDE_REG&&regs[t].regmap_entry[hr]>=64&&regs[t].regmap_entry[hr]<TEMPREG+64) {
4482         if(i_regmap[hr]!=regs[t].regmap_entry[hr]) {
4483           assert(regs[t].regmap_entry[hr]!=64);
4484           if((i_is32>>(regs[t].regmap_entry[hr]&63))&1) {
4485             int lr=get_reg(regs[t].regmap_entry,regs[t].regmap_entry[hr]-64);
4486             if(lr<0) {
4487               emit_loadreg(regs[t].regmap_entry[hr],hr);
4488             }
4489             else
4490             {
4491               emit_sarimm(lr,31,hr);
4492             }
4493           }
4494           else
4495           {
4496             emit_loadreg(regs[t].regmap_entry[hr],hr);
4497           }
4498         }
4499         else if((i_is32>>(regs[t].regmap_entry[hr]&63))&1) {
4500           int lr=get_reg(regs[t].regmap_entry,regs[t].regmap_entry[hr]-64);
4501           assert(lr>=0);
4502           emit_sarimm(lr,31,hr);
4503         }
4504       }
4505     }
4506   }
4507 }
4508
4509 static int match_bt(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr)
4510 {
4511   if(addr>=start && addr<start+slen*4-4)
4512   {
4513     int t=(addr-start)>>2;
4514     int hr;
4515     if(regs[t].regmap_entry[HOST_CCREG]!=CCREG) return 0;
4516     for(hr=0;hr<HOST_REGS;hr++)
4517     {
4518       if(hr!=EXCLUDE_REG)
4519       {
4520         if(i_regmap[hr]!=regs[t].regmap_entry[hr])
4521         {
4522           if(regs[t].regmap_entry[hr]>=0&&(regs[t].regmap_entry[hr]|64)<TEMPREG+64)
4523           {
4524             return 0;
4525           }
4526           else
4527           if((i_dirty>>hr)&1)
4528           {
4529             if(i_regmap[hr]<TEMPREG)
4530             {
4531               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4532                 return 0;
4533             }
4534             else if(i_regmap[hr]>=64&&i_regmap[hr]<TEMPREG+64)
4535             {
4536               if(!((unneeded_reg_upper[t]>>(i_regmap[hr]&63))&1))
4537                 return 0;
4538             }
4539           }
4540         }
4541         else // Same register but is it 32-bit or dirty?
4542         if(i_regmap[hr]>=0)
4543         {
4544           if(!((regs[t].dirty>>hr)&1))
4545           {
4546             if((i_dirty>>hr)&1)
4547             {
4548               if(!((unneeded_reg[t]>>i_regmap[hr])&1))
4549               {
4550                 //DebugMessage(M64MSG_VERBOSE, "%x: dirty no match",addr);
4551                 return 0;
4552               }
4553             }
4554           }
4555           if((((regs[t].was32^i_is32)&~unneeded_reg_upper[t])>>(i_regmap[hr]&63))&1)
4556           {
4557             //DebugMessage(M64MSG_VERBOSE, "%x: is32 no match",addr);
4558             return 0;
4559           }
4560         }
4561       }
4562     }
4563     //if(is32[t]&~unneeded_reg_upper[t]&~i_is32) return 0;
4564     if(requires_32bit[t]&~i_is32) return 0;
4565     // Delay slots are not valid branch targets
4566     //if(t>0&&(itype[t-1]==RJUMP||itype[t-1]==UJUMP||itype[t-1]==CJUMP||itype[t-1]==SJUMP||itype[t-1]==FJUMP)) return 0;
4567     // Delay slots require additional processing, so do not match
4568     if(is_ds[t]) return 0;
4569   }
4570   else
4571   {
4572     int hr;
4573     for(hr=0;hr<HOST_REGS;hr++)
4574     {
4575       if(hr!=EXCLUDE_REG)
4576       {
4577         if(i_regmap[hr]>=0)
4578         {
4579           if(hr!=HOST_CCREG||i_regmap[hr]!=CCREG)
4580           {
4581             if((i_dirty>>hr)&1)
4582             {
4583               return 0;
4584             }
4585           }
4586         }
4587       }
4588     }
4589   }
4590   return 1;
4591 }
4592
4593 // Used when a branch jumps into the delay slot of another branch
4594 static void ds_assemble_entry(int i)
4595 {
4596   int t=(ba[i]-start)>>2;
4597   if(!instr_addr[t]) instr_addr[t]=(u_int)out;
4598   assem_debug("Assemble delay slot at %x",ba[i]);
4599   assem_debug("<->");
4600   if(regs[t].regmap_entry[HOST_CCREG]==CCREG&&regs[t].regmap[HOST_CCREG]!=CCREG)
4601     wb_register(CCREG,regs[t].regmap_entry,regs[t].wasdirty,regs[t].was32);
4602   load_regs(regs[t].regmap_entry,regs[t].regmap,regs[t].was32,rs1[t],rs2[t]);
4603   address_generation(t,&regs[t],regs[t].regmap_entry);
4604   if(itype[t]==LOAD||itype[t]==LOADLR||itype[t]==STORE||itype[t]==STORELR||itype[t]==C1LS)
4605     load_regs(regs[t].regmap_entry,regs[t].regmap,regs[t].was32,MMREG,ROREG);
4606   if(itype[t]==STORE||itype[t]==STORELR||(opcode[t]&0x3b)==0x39)
4607     load_regs(regs[t].regmap_entry,regs[t].regmap,regs[t].was32,INVCP,INVCP);
4608   cop1_usable=0;
4609   is_delayslot=0;
4610   switch(itype[t]) {
4611     case ALU:
4612       alu_assemble(t,&regs[t]);break;
4613     case IMM16:
4614       imm16_assemble(t,&regs[t]);break;
4615     case SHIFT:
4616       shift_assemble(t,&regs[t]);break;
4617     case SHIFTIMM:
4618       shiftimm_assemble(t,&regs[t]);break;
4619     case LOAD:
4620       load_assemble(t,&regs[t]);break;
4621     case LOADLR:
4622       loadlr_assemble(t,&regs[t]);break;
4623     case STORE:
4624       store_assemble(t,&regs[t]);break;
4625     case STORELR:
4626       storelr_assemble(t,&regs[t]);break;
4627     case COP0:
4628       cop0_assemble(t,&regs[t]);break;
4629     case COP1:
4630       cop1_assemble(t,&regs[t]);break;
4631     case C1LS:
4632       c1ls_assemble(t,&regs[t]);break;
4633     case FCONV:
4634       fconv_assemble(t,&regs[t]);break;
4635     case FLOAT:
4636       float_assemble(t,&regs[t]);break;
4637     case FCOMP:
4638       fcomp_assemble(t,&regs[t]);break;
4639     case MULTDIV:
4640       multdiv_assemble(t,&regs[t]);break;
4641     case MOV:
4642       mov_assemble(t,&regs[t]);break;
4643     case SYSCALL:
4644     case SPAN:
4645     case UJUMP:
4646     case RJUMP:
4647     case CJUMP:
4648     case SJUMP:
4649     case FJUMP:
4650       DebugMessage(M64MSG_VERBOSE, "Jump in the delay slot.  This is probably a bug.");
4651   }
4652   store_regs_bt(regs[t].regmap,regs[t].is32,regs[t].dirty,ba[i]+4);
4653   load_regs_bt(regs[t].regmap,regs[t].is32,regs[t].dirty,ba[i]+4);
4654   if(internal_branch(regs[t].is32,ba[i]+4))
4655     assem_debug("branch: internal");
4656   else
4657     assem_debug("branch: external");
4658   assert(internal_branch(regs[t].is32,ba[i]+4));
4659   add_to_linker((int)out,ba[i]+4,internal_branch(regs[t].is32,ba[i]+4));
4660   emit_jmp(0);
4661 }
4662
4663 static void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert)
4664 {
4665   int count;
4666   int jaddr;
4667   int idle=0;
4668   if(itype[i]==RJUMP)
4669   {
4670     *adj=0;
4671   }
4672   //if(ba[i]>=start && ba[i]<(start+slen*4))
4673   if(internal_branch(branch_regs[i].is32,ba[i]))
4674   {
4675     int t=(ba[i]-start)>>2;
4676     if(is_ds[t]) *adj=-1; // Branch into delay slot adds an extra cycle
4677     else *adj=ccadj[t];
4678   }
4679   else
4680   {
4681     *adj=0;
4682   }
4683   count=ccadj[i];
4684   if(taken==TAKEN && i==(ba[i]-start)>>2 && source[i+1]==0) {
4685     // Idle loop
4686     if(count&1) emit_addimm_and_set_flags(2*(count+2),HOST_CCREG);
4687     idle=(int)out;
4688     //emit_subfrommem(&idlecount,HOST_CCREG); // Count idle cycles
4689     emit_andimm(HOST_CCREG,3,HOST_CCREG);
4690     jaddr=(int)out;
4691     emit_jmp(0);
4692   }
4693   else if(*adj==0||invert) {
4694     emit_addimm_and_set_flags(CLOCK_DIVIDER*(count+2),HOST_CCREG);
4695     jaddr=(int)out;
4696     emit_jns(0);
4697   }
4698   else
4699   {
4700     emit_cmpimm(HOST_CCREG,-CLOCK_DIVIDER*(count+2));
4701     jaddr=(int)out;
4702     emit_jns(0);
4703   }
4704   add_stub(CC_STUB,jaddr,idle?idle:(int)out,(*adj==0||invert||idle)?0:(count+2),i,addr,taken,0);
4705 }
4706
4707 static void do_ccstub(int n)
4708 {
4709   literal_pool(256);
4710   assem_debug("do_ccstub %x",start+stubs[n][4]*4);
4711   set_jump_target(stubs[n][1],(int)out);
4712   int i=stubs[n][4];
4713   if(stubs[n][6]==NULLDS) {
4714     // Delay slot instruction is nullified ("likely" branch)
4715     wb_dirtys(regs[i].regmap,regs[i].is32,regs[i].dirty);
4716   }
4717   else if(stubs[n][6]!=TAKEN) {
4718     wb_dirtys(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty);
4719   }
4720   else {
4721     if(internal_branch(branch_regs[i].is32,ba[i]))
4722       wb_needed_dirtys(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
4723   }
4724   if(stubs[n][5]!=-1)
4725   {
4726     // Save PC as return address
4727     emit_movimm(stubs[n][5],EAX);
4728     emit_writeword(EAX,(int)&pcaddr);
4729   }
4730   else
4731   {
4732     // Return address depends on which way the branch goes
4733     if(itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
4734     {
4735       int s1l=get_reg(branch_regs[i].regmap,rs1[i]);
4736       int s1h=get_reg(branch_regs[i].regmap,rs1[i]|64);
4737       int s2l=get_reg(branch_regs[i].regmap,rs2[i]);
4738       int s2h=get_reg(branch_regs[i].regmap,rs2[i]|64);
4739       if(rs1[i]==0)
4740       {
4741         s1l=s2l;s1h=s2h;
4742         s2l=s2h=-1;
4743       }
4744       else if(rs2[i]==0)
4745       {
4746         s2l=s2h=-1;
4747       }
4748       if((branch_regs[i].is32>>rs1[i])&(branch_regs[i].is32>>rs2[i])&1) {
4749         s1h=s2h=-1;
4750       }
4751       assert(s1l>=0);
4752       #ifdef DESTRUCTIVE_WRITEBACK
4753       if(rs1[i]) {
4754         if((branch_regs[i].dirty>>s1l)&(branch_regs[i].is32>>rs1[i])&1)
4755           emit_loadreg(rs1[i],s1l);
4756       }
4757       else {
4758         if((branch_regs[i].dirty>>s1l)&(branch_regs[i].is32>>rs2[i])&1)
4759           emit_loadreg(rs2[i],s1l);
4760       }
4761       if(s2l>=0)
4762         if((branch_regs[i].dirty>>s2l)&(branch_regs[i].is32>>rs2[i])&1)
4763           emit_loadreg(rs2[i],s2l);
4764       #endif
4765       int hr=0;
4766       int addr,alt,ntaddr;
4767       while(hr<HOST_REGS)
4768       {
4769         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4770            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4771            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4772         {
4773           addr=hr++;break;
4774         }
4775         hr++;
4776       }
4777       while(hr<HOST_REGS)
4778       {
4779         if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4780            (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4781            (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4782         {
4783           alt=hr++;break;
4784         }
4785         hr++;
4786       }
4787       if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
4788       {
4789         while(hr<HOST_REGS)
4790         {
4791           if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
4792              (branch_regs[i].regmap[hr]&63)!=rs1[i] &&
4793              (branch_regs[i].regmap[hr]&63)!=rs2[i] )
4794           {
4795             ntaddr=hr;break;
4796           }
4797           hr++;
4798         }
4799         assert(hr<HOST_REGS);
4800       }
4801       if((opcode[i]&0x2f)==4) // BEQ
4802       {
4803         #ifdef HAVE_CMOV_IMM
4804         if(s1h<0) {
4805           if(s2l>=0) emit_cmp(s1l,s2l);
4806           else emit_test(s1l,s1l);
4807           emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
4808         }
4809         else
4810         #endif
4811         {
4812           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4813           if(s1h>=0) {
4814             if(s2h>=0) emit_cmp(s1h,s2h);
4815             else emit_test(s1h,s1h);
4816             emit_cmovne_reg(alt,addr);
4817           }
4818           if(s2l>=0) emit_cmp(s1l,s2l);
4819           else emit_test(s1l,s1l);
4820           emit_cmovne_reg(alt,addr);
4821         }
4822       }
4823       if((opcode[i]&0x2f)==5) // BNE
4824       {
4825         #ifdef HAVE_CMOV_IMM
4826         if(s1h<0) {
4827           if(s2l>=0) emit_cmp(s1l,s2l);
4828           else emit_test(s1l,s1l);
4829           emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
4830         }
4831         else
4832         #endif
4833         {
4834           emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
4835           if(s1h>=0) {
4836             if(s2h>=0) emit_cmp(s1h,s2h);
4837             else emit_test(s1h,s1h);
4838             emit_cmovne_reg(alt,addr);
4839           }
4840           if(s2l>=0) emit_cmp(s1l,s2l);
4841           else emit_test(s1l,s1l);
4842           emit_cmovne_reg(alt,addr);
4843         }
4844       }
4845       if((opcode[i]&0x2f)==6) // BLEZ
4846       {
4847         //emit_movimm(ba[i],alt);
4848         //emit_movimm(start+i*4+8,addr);
4849         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4850         emit_cmpimm(s1l,1);
4851         if(s1h>=0) emit_mov(addr,ntaddr);
4852         emit_cmovl_reg(alt,addr);
4853         if(s1h>=0) {
4854           emit_test(s1h,s1h);
4855           emit_cmovne_reg(ntaddr,addr);
4856           emit_cmovs_reg(alt,addr);
4857         }
4858       }
4859       if((opcode[i]&0x2f)==7) // BGTZ
4860       {
4861         //emit_movimm(ba[i],addr);
4862         //emit_movimm(start+i*4+8,ntaddr);
4863         emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
4864         emit_cmpimm(s1l,1);
4865         if(s1h>=0) emit_mov(addr,alt);
4866         emit_cmovl_reg(ntaddr,addr);
4867         if(s1h>=0) {
4868           emit_test(s1h,s1h);
4869           emit_cmovne_reg(alt,addr);
4870           emit_cmovs_reg(ntaddr,addr);
4871         }
4872       }
4873       if((opcode[i]==1)&&(opcode2[i]&0x2D)==0) // BLTZ
4874       {
4875         //emit_movimm(ba[i],alt);
4876         //emit_movimm(start+i*4+8,addr);
4877         emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4878         if(s1h>=0) emit_test(s1h,s1h);
4879         else emit_test(s1l,s1l);
4880         emit_cmovs_reg(alt,addr);
4881       }
4882       if((opcode[i]==1)&&(opcode2[i]&0x2D)==1) // BGEZ
4883       {
4884         //emit_movimm(ba[i],addr);
4885         //emit_movimm(start+i*4+8,alt);
4886         emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4887         if(s1h>=0) emit_test(s1h,s1h);
4888         else emit_test(s1l,s1l);
4889         emit_cmovs_reg(alt,addr);
4890       }
4891       if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
4892         if(source[i]&0x10000) // BC1T
4893         {
4894           //emit_movimm(ba[i],alt);
4895           //emit_movimm(start+i*4+8,addr);
4896           emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
4897           emit_testimm(s1l,0x800000);
4898           emit_cmovne_reg(alt,addr);
4899         }
4900         else // BC1F
4901         {
4902           //emit_movimm(ba[i],addr);
4903           //emit_movimm(start+i*4+8,alt);
4904           emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
4905           emit_testimm(s1l,0x800000);
4906           emit_cmovne_reg(alt,addr);
4907         }
4908       }
4909       emit_writeword(addr,(int)&pcaddr);
4910     }
4911     else
4912     if(itype[i]==RJUMP)
4913     {
4914       int r=get_reg(branch_regs[i].regmap,rs1[i]);
4915       if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
4916         r=get_reg(branch_regs[i].regmap,RTEMP);
4917       }
4918       emit_writeword(r,(int)&pcaddr);
4919     }
4920     else {DebugMessage(M64MSG_ERROR, "Unknown branch type in do_ccstub");exit(1);}
4921   }
4922   // Update cycle count
4923   assert(branch_regs[i].regmap[HOST_CCREG]==CCREG||branch_regs[i].regmap[HOST_CCREG]==-1);
4924   if(stubs[n][3]) emit_addimm(HOST_CCREG,CLOCK_DIVIDER*stubs[n][3],HOST_CCREG);
4925   emit_call((int)cc_interrupt);
4926   if(stubs[n][3]) emit_addimm(HOST_CCREG,-CLOCK_DIVIDER*stubs[n][3],HOST_CCREG);
4927   if(stubs[n][6]==TAKEN) {
4928     if(internal_branch(branch_regs[i].is32,ba[i]))
4929       load_needed_regs(branch_regs[i].regmap,regs[(ba[i]-start)>>2].regmap_entry);
4930     else if(itype[i]==RJUMP) {
4931       if(get_reg(branch_regs[i].regmap,RTEMP)>=0)
4932         emit_readword((int)&pcaddr,get_reg(branch_regs[i].regmap,RTEMP));
4933       else
4934         emit_loadreg(rs1[i],get_reg(branch_regs[i].regmap,rs1[i]));
4935     }
4936   }else if(stubs[n][6]==NOTTAKEN) {
4937     if(i<slen-2) load_needed_regs(branch_regs[i].regmap,regmap_pre[i+2]);
4938     else load_all_regs(branch_regs[i].regmap);
4939   }else if(stubs[n][6]==NULLDS) {
4940     // Delay slot instruction is nullified ("likely" branch)
4941     if(i<slen-2) load_needed_regs(regs[i].regmap,regmap_pre[i+2]);
4942     else load_all_regs(regs[i].regmap);
4943   }else{
4944     load_all_regs(branch_regs[i].regmap);
4945   }
4946   emit_jmp(stubs[n][2]); // return address
4947
4948   /* This works but uses a lot of memory...
4949   emit_readword((int)&last_count,ECX);
4950   emit_add(HOST_CCREG,ECX,EAX);
4951   emit_writeword(EAX,(int)&Count);
4952   emit_call((int)gen_interupt);
4953   emit_readword((int)&Count,HOST_CCREG);
4954   emit_readword((int)&next_interupt,EAX);
4955   emit_readword((int)&pending_exception,EBX);
4956   emit_writeword(EAX,(int)&last_count);
4957   emit_sub(HOST_CCREG,EAX,HOST_CCREG);
4958   emit_test(EBX,EBX);
4959   int jne_instr=(int)out;
4960   emit_jne(0);
4961   if(stubs[n][3]) emit_addimm(HOST_CCREG,-2*stubs[n][3],HOST_CCREG);
4962   load_all_regs(branch_regs[i].regmap);
4963   emit_jmp(stubs[n][2]); // return address
4964   set_jump_target(jne_instr,(int)out);
4965   emit_readword((int)&pcaddr,EAX);
4966   // Call get_addr_ht instead of doing the hash table here.
4967   // This code is executed infrequently and takes up a lot of space
4968   // so smaller is better.
4969   emit_storereg(CCREG,HOST_CCREG);
4970   emit_pushreg(EAX);
4971   emit_call((int)get_addr_ht);
4972   emit_loadreg(CCREG,HOST_CCREG);
4973   emit_addimm(ESP,4,ESP);
4974   emit_jmpreg(EAX);*/
4975 }
4976
4977 static void add_to_linker(int addr,int target,int ext)
4978 {
4979   link_addr[linkcount][0]=addr;
4980   link_addr[linkcount][1]=target;
4981   link_addr[linkcount][2]=ext;
4982   linkcount++;
4983 }
4984
4985 static void ujump_assemble(int i,struct regstat *i_regs)
4986 {
4987   #ifdef REG_PREFETCH
4988   signed char *i_regmap=i_regs->regmap;
4989   #endif
4990   if(i==(ba[i]-start)>>2) assem_debug("idle loop");
4991   address_generation(i+1,i_regs,regs[i].regmap_entry);
4992   #ifdef REG_PREFETCH
4993   int temp=get_reg(branch_regs[i].regmap,PTEMP);
4994   if(rt1[i]==31&&temp>=0)
4995   {
4996     int return_address=start+i*4+8;
4997     if(get_reg(branch_regs[i].regmap,31)>0)
4998     if(i_regmap[temp]==PTEMP) emit_movimm((int)hash_table[((return_address>>16)^return_address)&0xFFFF],temp);
4999   }
5000   #endif
5001   ds_assemble(i+1,i_regs);
5002   uint64_t bc_unneeded=branch_regs[i].u;
5003   uint64_t bc_unneeded_upper=branch_regs[i].uu;
5004   bc_unneeded|=1|(1LL<<rt1[i]);
5005   bc_unneeded_upper|=1|(1LL<<rt1[i]);
5006   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5007                 bc_unneeded,bc_unneeded_upper);
5008   load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5009   if(rt1[i]==31) {
5010     int rt;
5011     unsigned int return_address;
5012     assert(rt1[i+1]!=31);
5013     assert(rt2[i+1]!=31);
5014     rt=get_reg(branch_regs[i].regmap,31);
5015     assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5016     //assert(rt>=0);
5017     return_address=start+i*4+8;
5018     if(rt>=0) {
5019       #ifdef USE_MINI_HT
5020       if(internal_branch(branch_regs[i].is32,return_address)) {
5021         int temp=rt+1;
5022         if(temp==EXCLUDE_REG||temp>=HOST_REGS||
5023            branch_regs[i].regmap[temp]>=0)
5024         {
5025           temp=get_reg(branch_regs[i].regmap,-1);
5026         }
5027         #ifdef HOST_TEMPREG
5028         if(temp<0) temp=HOST_TEMPREG;
5029         #endif
5030         if(temp>=0) do_miniht_insert(return_address,rt,temp);
5031         else emit_movimm(return_address,rt);
5032       }
5033       else
5034       #endif
5035       {
5036         #ifdef REG_PREFETCH
5037         if(temp>=0)
5038         {
5039           if(i_regmap[temp]!=PTEMP) emit_movimm((int)hash_table[((return_address>>16)^return_address)&0xFFFF],temp);
5040         }
5041         #endif
5042         emit_movimm(return_address,rt); // PC into link register
5043         #ifdef IMM_PREFETCH
5044         emit_prefetch(hash_table[((return_address>>16)^return_address)&0xFFFF]);
5045         #endif
5046       }
5047     }
5048   }
5049   int cc,adj;
5050   cc=get_reg(branch_regs[i].regmap,CCREG);
5051   assert(cc==HOST_CCREG);
5052   store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5053   #ifdef REG_PREFETCH
5054   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
5055   #endif
5056   do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5057   if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5058   load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5059   if(internal_branch(branch_regs[i].is32,ba[i]))
5060     assem_debug("branch: internal");
5061   else
5062     assem_debug("branch: external");
5063   if(internal_branch(branch_regs[i].is32,ba[i])&&is_ds[(ba[i]-start)>>2]) {
5064     ds_assemble_entry(i);
5065   }
5066   else {
5067     add_to_linker((int)out,ba[i],internal_branch(branch_regs[i].is32,ba[i]));
5068     emit_jmp(0);
5069   }
5070 }
5071
5072 static void rjump_assemble(int i,struct regstat *i_regs)
5073 {
5074   #ifdef REG_PREFETCH
5075   signed char *i_regmap=i_regs->regmap;
5076   #endif
5077   int temp;
5078   int rs,cc;
5079   rs=get_reg(branch_regs[i].regmap,rs1[i]);
5080   assert(rs>=0);
5081   if(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]) {
5082     // Delay slot abuse, make a copy of the branch address register
5083     temp=get_reg(branch_regs[i].regmap,RTEMP);
5084     assert(temp>=0);
5085     assert(regs[i].regmap[temp]==RTEMP);
5086     emit_mov(rs,temp);
5087     rs=temp;
5088   }
5089   address_generation(i+1,i_regs,regs[i].regmap_entry);
5090   #ifdef REG_PREFETCH
5091   if(rt1[i]==31)
5092   {
5093     if((temp=get_reg(branch_regs[i].regmap,PTEMP))>=0) {
5094       int return_address=start+i*4+8;
5095       if(i_regmap[temp]==PTEMP) emit_movimm((int)hash_table[((return_address>>16)^return_address)&0xFFFF],temp);
5096     }
5097   }
5098   #endif
5099   #ifdef USE_MINI_HT
5100   if(rs1[i]==31) {
5101     int rh=get_reg(regs[i].regmap,RHASH);
5102     if(rh>=0) do_preload_rhash(rh);
5103   }
5104   #endif
5105   ds_assemble(i+1,i_regs);
5106   uint64_t bc_unneeded=branch_regs[i].u;
5107   uint64_t bc_unneeded_upper=branch_regs[i].uu;
5108   bc_unneeded|=1|(1LL<<rt1[i]);
5109   bc_unneeded_upper|=1|(1LL<<rt1[i]);
5110   bc_unneeded&=~(1LL<<rs1[i]);
5111   wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5112                 bc_unneeded,bc_unneeded_upper);
5113   load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i],CCREG);
5114   if(rt1[i]!=0) {
5115     int rt,return_address;
5116     assert(rt1[i+1]!=rt1[i]);
5117     assert(rt2[i+1]!=rt1[i]);
5118     rt=get_reg(branch_regs[i].regmap,rt1[i]);
5119     assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5120     assert(rt>=0);
5121     return_address=start+i*4+8;
5122     #ifdef REG_PREFETCH
5123     if(temp>=0)
5124     {
5125       if(i_regmap[temp]!=PTEMP) emit_movimm((int)hash_table[((return_address>>16)^return_address)&0xFFFF],temp);
5126     }
5127     #endif
5128     emit_movimm(return_address,rt); // PC into link register
5129     #ifdef IMM_PREFETCH
5130     emit_prefetch(hash_table[((return_address>>16)^return_address)&0xFFFF]);
5131     #endif
5132   }
5133   cc=get_reg(branch_regs[i].regmap,CCREG);
5134   assert(cc==HOST_CCREG);
5135   #ifdef USE_MINI_HT
5136   int rh=get_reg(branch_regs[i].regmap,RHASH);
5137   int ht=get_reg(branch_regs[i].regmap,RHTBL);
5138   if(rs1[i]==31) {
5139     if(regs[i].regmap[rh]!=RHASH) do_preload_rhash(rh);
5140     do_preload_rhtbl(ht);
5141     do_rhash(rs,rh);
5142   }
5143   #endif
5144   store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,-1);
5145   #ifdef DESTRUCTIVE_WRITEBACK
5146   if((branch_regs[i].dirty>>rs)&(branch_regs[i].is32>>rs1[i])&1) {
5147     if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
5148       emit_loadreg(rs1[i],rs);
5149     }
5150   }
5151   #endif
5152   #ifdef REG_PREFETCH
5153   if(rt1[i]==31&&temp>=0) emit_prefetchreg(temp);
5154   #endif
5155   #ifdef USE_MINI_HT
5156   if(rs1[i]==31) {
5157     do_miniht_load(ht,rh);
5158   }
5159   #endif
5160   //do_cc(i,branch_regs[i].regmap,&adj,-1,TAKEN);
5161   //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen
5162   //assert(adj==0);
5163   emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),HOST_CCREG);
5164   add_stub(CC_STUB,(int)out,jump_vaddr_reg[rs],0,i,-1,TAKEN,0);
5165   emit_jns(0);
5166   //load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,-1);
5167   #ifdef USE_MINI_HT
5168   if(rs1[i]==31) {
5169     do_miniht_jump(rs,rh,ht);
5170   }
5171   else
5172   #endif
5173   {
5174     //if(rs!=EAX) emit_mov(rs,EAX);
5175     //emit_jmp((int)jump_vaddr_eax);
5176     emit_jmp(jump_vaddr_reg[rs]);
5177   }
5178   /* Check hash table
5179   temp=!rs;
5180   emit_mov(rs,temp);
5181   emit_shrimm(rs,16,rs);
5182   emit_xor(temp,rs,rs);
5183   emit_movzwl_reg(rs,rs);
5184   emit_shlimm(rs,4,rs);
5185   emit_cmpmem_indexed((int)hash_table,rs,temp);
5186   emit_jne((int)out+14);
5187   emit_readword_indexed((int)hash_table+4,rs,rs);
5188   emit_jmpreg(rs);
5189   emit_cmpmem_indexed((int)hash_table+8,rs,temp);
5190   emit_addimm_no_flags(8,rs);
5191   emit_jeq((int)out-17);
5192   // No hit on hash table, call compiler
5193   emit_pushreg(temp);
5194 //DEBUG >
5195 #ifdef DEBUG_CYCLE_COUNT
5196   emit_readword((int)&last_count,ECX);
5197   emit_add(HOST_CCREG,ECX,HOST_CCREG);
5198   emit_readword((int)&next_interupt,ECX);
5199   emit_writeword(HOST_CCREG,(int)&Count);
5200   emit_sub(HOST_CCREG,ECX,HOST_CCREG);
5201   emit_writeword(ECX,(int)&last_count);
5202 #endif
5203 //DEBUG <
5204   emit_storereg(CCREG,HOST_CCREG);
5205   emit_call((int)get_addr);
5206   emit_loadreg(CCREG,HOST_CCREG);
5207   emit_addimm(ESP,4,ESP);
5208   emit_jmpreg(EAX);*/
5209   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5210   if(rt1[i]!=31&&i<slen-2&&(((u_int)out)&7)) emit_mov(13,13);
5211   #endif
5212 }
5213
5214 static void cjump_assemble(int i,struct regstat *i_regs)
5215 {
5216   signed char *i_regmap=i_regs->regmap;
5217   int cc;
5218   int match;
5219   match=match_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5220   assem_debug("match=%d",match);
5221   int s1h,s1l,s2h,s2l;
5222   int prev_cop1_usable=cop1_usable;
5223   int unconditional=0,nop=0;
5224   int only32=0;
5225   int invert=0;
5226   int internal=internal_branch(branch_regs[i].is32,ba[i]);
5227   if(i==(ba[i]-start)>>2) assem_debug("idle loop");
5228   if(!match) invert=1;
5229   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5230   if(i>(ba[i]-start)>>2) invert=1;
5231   #endif
5232
5233   if(ooo[i]) {
5234     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
5235     s1h=get_reg(branch_regs[i].regmap,rs1[i]|64);
5236     s2l=get_reg(branch_regs[i].regmap,rs2[i]);
5237     s2h=get_reg(branch_regs[i].regmap,rs2[i]|64);
5238   }
5239   else {
5240     s1l=get_reg(i_regmap,rs1[i]);
5241     s1h=get_reg(i_regmap,rs1[i]|64);
5242     s2l=get_reg(i_regmap,rs2[i]);
5243     s2h=get_reg(i_regmap,rs2[i]|64);
5244   }
5245   if(rs1[i]==0&&rs2[i]==0)
5246   {
5247     if(opcode[i]&1) nop=1;
5248     else unconditional=1;
5249     //assert(opcode[i]!=5);
5250     //assert(opcode[i]!=7);
5251     //assert(opcode[i]!=0x15);
5252     //assert(opcode[i]!=0x17);
5253   }
5254   else if(rs1[i]==0)
5255   {
5256     s1l=s2l;s1h=s2h;
5257     s2l=s2h=-1;
5258     only32=(regs[i].was32>>rs2[i])&1;
5259   }
5260   else if(rs2[i]==0)
5261   {
5262     s2l=s2h=-1;
5263     only32=(regs[i].was32>>rs1[i])&1;
5264   }
5265   else {
5266     only32=(regs[i].was32>>rs1[i])&(regs[i].was32>>rs2[i])&1;
5267   }
5268
5269   if(ooo[i]) {
5270     // Out of order execution (delay slot first)
5271     //DebugMessage(M64MSG_VERBOSE, "OOOE");
5272     address_generation(i+1,i_regs,regs[i].regmap_entry);
5273     ds_assemble(i+1,i_regs);
5274     int adj;
5275     uint64_t bc_unneeded=branch_regs[i].u;
5276     uint64_t bc_unneeded_upper=branch_regs[i].uu;
5277     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5278     bc_unneeded_upper&=~((1LL<<us1[i])|(1LL<<us2[i]));
5279     bc_unneeded|=1;
5280     bc_unneeded_upper|=1;
5281     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5282                   bc_unneeded,bc_unneeded_upper);
5283     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i],rs2[i]);
5284     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5285     cc=get_reg(branch_regs[i].regmap,CCREG);
5286     assert(cc==HOST_CCREG);
5287     if(unconditional)
5288       store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5289     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
5290     //assem_debug("cycle count (adj)");
5291     if(unconditional) {
5292       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5293       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
5294         if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5295         load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5296         if(internal)
5297           assem_debug("branch: internal");
5298         else
5299           assem_debug("branch: external");
5300         if(internal&&is_ds[(ba[i]-start)>>2]) {
5301           ds_assemble_entry(i);
5302         }
5303         else {
5304           add_to_linker((int)out,ba[i],internal);
5305           emit_jmp(0);
5306         }
5307         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5308         if(((u_int)out)&7) emit_addnop(0);
5309         #endif
5310       }
5311     }
5312     else if(nop) {
5313       emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),cc);
5314       int jaddr=(int)out;
5315       emit_jns(0);
5316       add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,NOTTAKEN,0);
5317     }
5318     else {
5319       int taken=0,nottaken=0,nottaken1=0;
5320       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5321       if(adj&&!invert) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5322       if(!only32)
5323       {
5324         assert(s1h>=0);
5325         if(opcode[i]==4) // BEQ
5326         {
5327           if(s2h>=0) emit_cmp(s1h,s2h);
5328           else emit_test(s1h,s1h);
5329           nottaken1=(int)out;
5330           emit_jne(1);
5331         }
5332         if(opcode[i]==5) // BNE
5333         {
5334           if(s2h>=0) emit_cmp(s1h,s2h);
5335           else emit_test(s1h,s1h);
5336           if(invert) taken=(int)out;
5337           else add_to_linker((int)out,ba[i],internal);
5338           emit_jne(0);
5339         }
5340         if(opcode[i]==6) // BLEZ
5341         {
5342           emit_test(s1h,s1h);
5343 //          emit_testimm(s1h,0);
5344           if(invert) taken=(int)out;
5345           else add_to_linker((int)out,ba[i],internal);
5346           emit_js(0);
5347           nottaken1=(int)out;
5348           emit_jne(1);
5349         }
5350         if(opcode[i]==7) // BGTZ
5351         {
5352           emit_test(s1h,s1h);
5353 //          emit_testimm(s1h,0);
5354           nottaken1=(int)out;
5355           emit_js(1);
5356           if(invert) taken=(int)out;
5357           else add_to_linker((int)out,ba[i],internal);
5358           emit_jne(0);
5359         }
5360       } // if(!only32)
5361
5362       //DebugMessage(M64MSG_VERBOSE, "branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5363       assert(s1l>=0);
5364       if(opcode[i]==4) // BEQ
5365       {
5366         if(s2l>=0) emit_cmp(s1l,s2l);
5367         else emit_test(s1l,s1l);
5368         if(invert){
5369           nottaken=(int)out;
5370           emit_jne(1);
5371         }else{
5372           add_to_linker((int)out,ba[i],internal);
5373           emit_jeq(0);
5374         }
5375       }
5376       if(opcode[i]==5) // BNE
5377       {
5378         if(s2l>=0) emit_cmp(s1l,s2l);
5379         else emit_test(s1l,s1l);
5380         if(invert){
5381           nottaken=(int)out;
5382           emit_jeq(1);
5383         }else{
5384           add_to_linker((int)out,ba[i],internal);
5385           emit_jne(0);
5386         }
5387       }
5388       if(opcode[i]==6) // BLEZ
5389       {
5390         emit_cmpimm(s1l,1);
5391         if(invert){
5392           nottaken=(int)out;
5393           emit_jge(1);
5394         }else{
5395           add_to_linker((int)out,ba[i],internal);
5396           emit_jl(0);
5397         }
5398       }
5399       if(opcode[i]==7) // BGTZ
5400       {
5401         emit_cmpimm(s1l,1);
5402         if(invert){
5403           nottaken=(int)out;
5404           emit_jl(1);
5405         }else{
5406           add_to_linker((int)out,ba[i],internal);
5407           emit_jge(0);
5408         }
5409       }
5410       if(invert) {
5411         if(taken) set_jump_target(taken,(int)out);
5412         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5413         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5414           if(adj) {
5415             emit_addimm(cc,-CLOCK_DIVIDER*adj,cc);
5416             add_to_linker((int)out,ba[i],internal);
5417           }else{
5418             emit_addnop(13);
5419             add_to_linker((int)out,ba[i],internal*2);
5420           }
5421           emit_jmp(0);
5422         }else
5423         #endif
5424         {
5425           if(adj) emit_addimm(cc,-CLOCK_DIVIDER*adj,cc);
5426           store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5427           load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5428           if(internal)
5429             assem_debug("branch: internal");
5430           else
5431             assem_debug("branch: external");
5432           if(internal&&is_ds[(ba[i]-start)>>2]) {
5433             ds_assemble_entry(i);
5434           }
5435           else {
5436             add_to_linker((int)out,ba[i],internal);
5437             emit_jmp(0);
5438           }
5439         }
5440         set_jump_target(nottaken,(int)out);
5441       }
5442
5443       if(nottaken1) set_jump_target(nottaken1,(int)out);
5444       if(adj) {
5445         if(!invert) emit_addimm(cc,CLOCK_DIVIDER*adj,cc);
5446       }
5447     } // (!unconditional)
5448   } // if(ooo)
5449   else
5450   {
5451     // In-order execution (branch first)
5452     //if(likely[i]) DebugMessage(M64MSG_VERBOSE, "IOL");
5453     //else
5454     //DebugMessage(M64MSG_VERBOSE, "IOE");
5455     int taken=0,nottaken=0,nottaken1=0;
5456     if(!unconditional&&!nop) {
5457       if(!only32)
5458       {
5459         assert(s1h>=0);
5460         if((opcode[i]&0x2f)==4) // BEQ
5461         {
5462           if(s2h>=0) emit_cmp(s1h,s2h);
5463           else emit_test(s1h,s1h);
5464           nottaken1=(int)out;
5465           emit_jne(2);
5466         }
5467         if((opcode[i]&0x2f)==5) // BNE
5468         {
5469           if(s2h>=0) emit_cmp(s1h,s2h);
5470           else emit_test(s1h,s1h);
5471           taken=(int)out;
5472           emit_jne(1);
5473         }
5474         if((opcode[i]&0x2f)==6) // BLEZ
5475         {
5476           emit_test(s1h,s1h);
5477           taken=(int)out;
5478           emit_js(1);
5479           nottaken1=(int)out;
5480           emit_jne(2);
5481         }
5482         if((opcode[i]&0x2f)==7) // BGTZ
5483         {
5484           emit_test(s1h,s1h);
5485           nottaken1=(int)out;
5486           emit_js(2);
5487           taken=(int)out;
5488           emit_jne(1);
5489         }
5490       } // if(!only32)
5491
5492       //DebugMessage(M64MSG_VERBOSE, "branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5493       assert(s1l>=0);
5494       if((opcode[i]&0x2f)==4) // BEQ
5495       {
5496         if(s2l>=0) emit_cmp(s1l,s2l);
5497         else emit_test(s1l,s1l);
5498         nottaken=(int)out;
5499         emit_jne(2);
5500       }
5501       if((opcode[i]&0x2f)==5) // BNE
5502       {
5503         if(s2l>=0) emit_cmp(s1l,s2l);
5504         else emit_test(s1l,s1l);
5505         nottaken=(int)out;
5506         emit_jeq(2);
5507       }
5508       if((opcode[i]&0x2f)==6) // BLEZ
5509       {
5510         emit_cmpimm(s1l,1);
5511         nottaken=(int)out;
5512         emit_jge(2);
5513       }
5514       if((opcode[i]&0x2f)==7) // BGTZ
5515       {
5516         emit_cmpimm(s1l,1);
5517         nottaken=(int)out;
5518         emit_jl(2);
5519       }
5520     } // if(!unconditional)
5521     int adj;
5522     uint64_t ds_unneeded=branch_regs[i].u;
5523     uint64_t ds_unneeded_upper=branch_regs[i].uu;
5524     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5525     ds_unneeded_upper&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
5526     if((~ds_unneeded_upper>>rt1[i+1])&1) ds_unneeded_upper&=~((1LL<<dep1[i+1])|(1LL<<dep2[i+1]));
5527     ds_unneeded|=1;
5528     ds_unneeded_upper|=1;
5529     // branch taken
5530     if(!nop) {
5531       if(taken) set_jump_target(taken,(int)out);
5532       assem_debug("1:");
5533       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5534                     ds_unneeded,ds_unneeded_upper);
5535       // load regs
5536       load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
5537       address_generation(i+1,&branch_regs[i],0);
5538       load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,INVCP);
5539       ds_assemble(i+1,&branch_regs[i]);
5540       cc=get_reg(branch_regs[i].regmap,CCREG);
5541       if(cc==-1) {
5542         emit_loadreg(CCREG,cc=HOST_CCREG);
5543         // CHECK: Is the following instruction (fall thru) allocated ok?
5544       }
5545       assert(cc==HOST_CCREG);
5546       store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5547       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5548       assem_debug("cycle count (adj)");
5549       if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5550       load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5551       if(internal)
5552         assem_debug("branch: internal");
5553       else
5554         assem_debug("branch: external");
5555       if(internal&&is_ds[(ba[i]-start)>>2]) {
5556         ds_assemble_entry(i);
5557       }
5558       else {
5559         add_to_linker((int)out,ba[i],internal);
5560         emit_jmp(0);
5561       }
5562     }
5563     // branch not taken
5564     cop1_usable=prev_cop1_usable;
5565     if(!unconditional) {
5566       if(nottaken1) set_jump_target(nottaken1,(int)out);
5567       set_jump_target(nottaken,(int)out);
5568       assem_debug("2:");
5569       if(!likely[i]) {
5570         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5571                       ds_unneeded,ds_unneeded_upper);
5572         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
5573         address_generation(i+1,&branch_regs[i],0);
5574         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5575         ds_assemble(i+1,&branch_regs[i]);
5576       }
5577       cc=get_reg(branch_regs[i].regmap,CCREG);
5578       if(cc==-1&&!likely[i]) {
5579         // Cycle count isn't in a register, temporarily load it then write it out
5580         emit_loadreg(CCREG,HOST_CCREG);
5581         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),HOST_CCREG);
5582         int jaddr=(int)out;
5583         emit_jns(0);
5584         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,NOTTAKEN,0);
5585         emit_storereg(CCREG,HOST_CCREG);
5586       }
5587       else{
5588         cc=get_reg(i_regmap,CCREG);
5589         assert(cc==HOST_CCREG);
5590         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),cc);
5591         int jaddr=(int)out;
5592         emit_jns(0);
5593         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5594       }
5595     }
5596   }
5597 }
5598
5599 static void sjump_assemble(int i,struct regstat *i_regs)
5600 {
5601   signed char *i_regmap=i_regs->regmap;
5602   int cc;
5603   int match;
5604   match=match_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5605   assem_debug("smatch=%d",match);
5606   int s1h,s1l;
5607   int prev_cop1_usable=cop1_usable;
5608   int unconditional=0,nevertaken=0;
5609   int only32=0;
5610   int invert=0;
5611   int internal=internal_branch(branch_regs[i].is32,ba[i]);
5612   if(i==(ba[i]-start)>>2) assem_debug("idle loop");
5613   if(!match) invert=1;
5614   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5615   if(i>(ba[i]-start)>>2) invert=1;
5616   #endif
5617
5618   //if(opcode2[i]>=0x10) return; // FIXME (BxxZAL)
5619   assert(opcode2[i]<0x10||rs1[i]==0); // FIXME (BxxZAL)
5620
5621   if(ooo[i]) {
5622     s1l=get_reg(branch_regs[i].regmap,rs1[i]);
5623     s1h=get_reg(branch_regs[i].regmap,rs1[i]|64);
5624   }
5625   else {
5626     s1l=get_reg(i_regmap,rs1[i]);
5627     s1h=get_reg(i_regmap,rs1[i]|64);
5628   }
5629   if(rs1[i]==0)
5630   {
5631     if(opcode2[i]&1) unconditional=1;
5632     else nevertaken=1;
5633     // These are never taken (r0 is never less than zero)
5634     //assert(opcode2[i]!=0);
5635     //assert(opcode2[i]!=2);
5636     //assert(opcode2[i]!=0x10);
5637     //assert(opcode2[i]!=0x12);
5638   }
5639   else {
5640     only32=(regs[i].was32>>rs1[i])&1;
5641   }
5642
5643   if(ooo[i]) {
5644     // Out of order execution (delay slot first)
5645     //DebugMessage(M64MSG_VERBOSE, "OOOE");
5646     address_generation(i+1,i_regs,regs[i].regmap_entry);
5647     ds_assemble(i+1,i_regs);
5648     int adj;
5649     uint64_t bc_unneeded=branch_regs[i].u;
5650     uint64_t bc_unneeded_upper=branch_regs[i].uu;
5651     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5652     bc_unneeded_upper&=~((1LL<<us1[i])|(1LL<<us2[i]));
5653     bc_unneeded|=1;
5654     bc_unneeded_upper|=1;
5655     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5656                   bc_unneeded,bc_unneeded_upper);
5657     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i],rs1[i]);
5658     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5659     if(rt1[i]==31) {
5660       int rt,return_address;
5661       assert(rt1[i+1]!=31);
5662       assert(rt2[i+1]!=31);
5663       rt=get_reg(branch_regs[i].regmap,31);
5664       assem_debug("branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5665       if(rt>=0) {
5666         // Save the PC even if the branch is not taken
5667         return_address=start+i*4+8;
5668         emit_movimm(return_address,rt); // PC into link register
5669         #ifdef IMM_PREFETCH
5670         if(!nevertaken) emit_prefetch(hash_table[((return_address>>16)^return_address)&0xFFFF]);
5671         #endif
5672       }
5673     }
5674     cc=get_reg(branch_regs[i].regmap,CCREG);
5675     assert(cc==HOST_CCREG);
5676     if(unconditional)
5677       store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5678     //do_cc(i,branch_regs[i].regmap,&adj,unconditional?ba[i]:-1,unconditional);
5679     assem_debug("cycle count (adj)");
5680     if(unconditional) {
5681       do_cc(i,branch_regs[i].regmap,&adj,ba[i],TAKEN,0);
5682       if(i!=(ba[i]-start)>>2 || source[i+1]!=0) {
5683         if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5684         load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5685         if(internal)
5686           assem_debug("branch: internal");
5687         else
5688           assem_debug("branch: external");
5689         if(internal&&is_ds[(ba[i]-start)>>2]) {
5690           ds_assemble_entry(i);
5691         }
5692         else {
5693           add_to_linker((int)out,ba[i],internal);
5694           emit_jmp(0);
5695         }
5696         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5697         if(((u_int)out)&7) emit_addnop(0);
5698         #endif
5699       }
5700     }
5701     else if(nevertaken) {
5702       emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),cc);
5703       int jaddr=(int)out;
5704       emit_jns(0);
5705       add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,NOTTAKEN,0);
5706     }
5707     else {
5708       int nottaken=0;
5709       do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5710       if(adj&&!invert) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5711       if(!only32)
5712       {
5713         assert(s1h>=0);
5714         if(opcode2[i]==0) // BLTZ
5715         {
5716           emit_test(s1h,s1h);
5717           if(invert){
5718             nottaken=(int)out;
5719             emit_jns(1);
5720           }else{
5721             add_to_linker((int)out,ba[i],internal);
5722             emit_js(0);
5723           }
5724         }
5725         if(opcode2[i]==1) // BGEZ
5726         {
5727           emit_test(s1h,s1h);
5728           if(invert){
5729             nottaken=(int)out;
5730             emit_js(1);
5731           }else{
5732             add_to_linker((int)out,ba[i],internal);
5733             emit_jns(0);
5734           }
5735         }
5736       } // if(!only32)
5737       else
5738       {
5739         assert(s1l>=0);
5740         if(opcode2[i]==0) // BLTZ
5741         {
5742           emit_test(s1l,s1l);
5743           if(invert){
5744             nottaken=(int)out;
5745             emit_jns(1);
5746           }else{
5747             add_to_linker((int)out,ba[i],internal);
5748             emit_js(0);
5749           }
5750         }
5751         if(opcode2[i]==1) // BGEZ
5752         {
5753           emit_test(s1l,s1l);
5754           if(invert){
5755             nottaken=(int)out;
5756             emit_js(1);
5757           }else{
5758             add_to_linker((int)out,ba[i],internal);
5759             emit_jns(0);
5760           }
5761         }
5762       } // if(!only32)
5763
5764       if(invert) {
5765         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5766         if(match&&(!internal||!is_ds[(ba[i]-start)>>2])) {
5767           if(adj) {
5768             emit_addimm(cc,-CLOCK_DIVIDER*adj,cc);
5769             add_to_linker((int)out,ba[i],internal);
5770           }else{
5771             emit_addnop(13);
5772             add_to_linker((int)out,ba[i],internal*2);
5773           }
5774           emit_jmp(0);
5775         }else
5776         #endif
5777         {
5778           if(adj) emit_addimm(cc,-CLOCK_DIVIDER*adj,cc);
5779           store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5780           load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5781           if(internal)
5782             assem_debug("branch: internal");
5783           else
5784             assem_debug("branch: external");
5785           if(internal&&is_ds[(ba[i]-start)>>2]) {
5786             ds_assemble_entry(i);
5787           }
5788           else {
5789             add_to_linker((int)out,ba[i],internal);
5790             emit_jmp(0);
5791           }
5792         }
5793         set_jump_target(nottaken,(int)out);
5794       }
5795
5796       if(adj) {
5797         if(!invert) emit_addimm(cc,CLOCK_DIVIDER*adj,cc);
5798       }
5799     } // (!unconditional)
5800   } // if(ooo)
5801   else
5802   {
5803     // In-order execution (branch first)
5804     //DebugMessage(M64MSG_VERBOSE, "IOE");
5805     int nottaken=0;
5806     if(!unconditional) {
5807       //DebugMessage(M64MSG_VERBOSE, "branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
5808       if(!only32)
5809       {
5810         assert(s1h>=0);
5811         if((opcode2[i]&0x1d)==0) // BLTZ/BLTZL
5812         {
5813           emit_test(s1h,s1h);
5814           nottaken=(int)out;
5815           emit_jns(1);
5816         }
5817         if((opcode2[i]&0x1d)==1) // BGEZ/BGEZL
5818         {
5819           emit_test(s1h,s1h);
5820           nottaken=(int)out;
5821           emit_js(1);
5822         }
5823       } // if(!only32)
5824       else
5825       {
5826         assert(s1l>=0);
5827         if((opcode2[i]&0x1d)==0) // BLTZ/BLTZL
5828         {
5829           emit_test(s1l,s1l);
5830           nottaken=(int)out;
5831           emit_jns(1);
5832         }
5833         if((opcode2[i]&0x1d)==1) // BGEZ/BGEZL
5834         {
5835           emit_test(s1l,s1l);
5836           nottaken=(int)out;
5837           emit_js(1);
5838         }
5839       }
5840     } // if(!unconditional)
5841     int adj;
5842     uint64_t ds_unneeded=branch_regs[i].u;
5843     uint64_t ds_unneeded_upper=branch_regs[i].uu;
5844     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
5845     ds_unneeded_upper&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
5846     if((~ds_unneeded_upper>>rt1[i+1])&1) ds_unneeded_upper&=~((1LL<<dep1[i+1])|(1LL<<dep2[i+1]));
5847     ds_unneeded|=1;
5848     ds_unneeded_upper|=1;
5849     // branch taken
5850     if(!nevertaken) {
5851       //assem_debug("1:");
5852       wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5853                     ds_unneeded,ds_unneeded_upper);
5854       // load regs
5855       load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
5856       address_generation(i+1,&branch_regs[i],0);
5857       load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,INVCP);
5858       ds_assemble(i+1,&branch_regs[i]);
5859       cc=get_reg(branch_regs[i].regmap,CCREG);
5860       if(cc==-1) {
5861         emit_loadreg(CCREG,cc=HOST_CCREG);
5862         // CHECK: Is the following instruction (fall thru) allocated ok?
5863       }
5864       assert(cc==HOST_CCREG);
5865       store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5866       do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
5867       assem_debug("cycle count (adj)");
5868       if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5869       load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5870       if(internal)
5871         assem_debug("branch: internal");
5872       else
5873         assem_debug("branch: external");
5874       if(internal&&is_ds[(ba[i]-start)>>2]) {
5875         ds_assemble_entry(i);
5876       }
5877       else {
5878         add_to_linker((int)out,ba[i],internal);
5879         emit_jmp(0);
5880       }
5881     }
5882     // branch not taken
5883     cop1_usable=prev_cop1_usable;
5884     if(!unconditional) {
5885       set_jump_target(nottaken,(int)out);
5886       assem_debug("1:");
5887       if(!likely[i]) {
5888         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5889                       ds_unneeded,ds_unneeded_upper);
5890         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
5891         address_generation(i+1,&branch_regs[i],0);
5892         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5893         ds_assemble(i+1,&branch_regs[i]);
5894       }
5895       cc=get_reg(branch_regs[i].regmap,CCREG);
5896       if(cc==-1&&!likely[i]) {
5897         // Cycle count isn't in a register, temporarily load it then write it out
5898         emit_loadreg(CCREG,HOST_CCREG);
5899         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),HOST_CCREG);
5900         int jaddr=(int)out;
5901         emit_jns(0);
5902         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,NOTTAKEN,0);
5903         emit_storereg(CCREG,HOST_CCREG);
5904       }
5905       else{
5906         cc=get_reg(i_regmap,CCREG);
5907         assert(cc==HOST_CCREG);
5908         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),cc);
5909         int jaddr=(int)out;
5910         emit_jns(0);
5911         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
5912       }
5913     }
5914   }
5915 }
5916
5917 static void fjump_assemble(int i,struct regstat *i_regs)
5918 {
5919   signed char *i_regmap=i_regs->regmap;
5920   int cc;
5921   int match;
5922   match=match_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
5923   assem_debug("fmatch=%d",match);
5924   int fs,cs;
5925   int eaddr;
5926   int invert=0;
5927   int internal=internal_branch(branch_regs[i].is32,ba[i]);
5928   if(i==(ba[i]-start)>>2) assem_debug("idle loop");
5929   if(!match) invert=1;
5930   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
5931   if(i>(ba[i]-start)>>2) invert=1;
5932   #endif
5933
5934   if(ooo[i]) {
5935     fs=get_reg(branch_regs[i].regmap,FSREG);
5936     address_generation(i+1,i_regs,regs[i].regmap_entry); // Is this okay?
5937   }
5938   else {
5939     fs=get_reg(i_regmap,FSREG);
5940   }
5941
5942   // Check cop1 unusable
5943   if(!cop1_usable) {
5944     cs=get_reg(i_regmap,CSREG);
5945     assert(cs>=0);
5946     emit_testimm(cs,0x20000000);
5947     eaddr=(int)out;
5948     emit_jeq(0);
5949     add_stub(FP_STUB,eaddr,(int)out,i,cs,(int)i_regs,0,0);
5950     cop1_usable=1;
5951   }
5952
5953   if(ooo[i]) {
5954     // Out of order execution (delay slot first)
5955     //DebugMessage(M64MSG_VERBOSE, "OOOE");
5956     ds_assemble(i+1,i_regs);
5957     int adj;
5958     uint64_t bc_unneeded=branch_regs[i].u;
5959     uint64_t bc_unneeded_upper=branch_regs[i].uu;
5960     bc_unneeded&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
5961     bc_unneeded_upper&=~((1LL<<us1[i])|(1LL<<us2[i]));
5962     bc_unneeded|=1;
5963     bc_unneeded_upper|=1;
5964     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
5965                   bc_unneeded,bc_unneeded_upper);
5966     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i],rs1[i]);
5967     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
5968     cc=get_reg(branch_regs[i].regmap,CCREG);
5969     assert(cc==HOST_CCREG);
5970     do_cc(i,branch_regs[i].regmap,&adj,-1,0,invert);
5971     assem_debug("cycle count (adj)");
5972     if(1) {
5973       int nottaken=0;
5974       if(adj&&!invert) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
5975       if(1) {
5976         assert(fs>=0);
5977         emit_testimm(fs,0x800000);
5978         if(source[i]&0x10000) // BC1T
5979         {
5980           if(invert){
5981             nottaken=(int)out;
5982             emit_jeq(1);
5983           }else{
5984             add_to_linker((int)out,ba[i],internal);
5985             emit_jne(0);
5986           }
5987         }
5988         else // BC1F
5989           if(invert){
5990             nottaken=(int)out;
5991             emit_jne(1);
5992           }else{
5993             add_to_linker((int)out,ba[i],internal);
5994             emit_jeq(0);
5995           }
5996         {
5997         }
5998       } // if(!only32)
5999
6000       if(invert) {
6001         if(adj) emit_addimm(cc,-CLOCK_DIVIDER*adj,cc);
6002         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
6003         else if(match) emit_addnop(13);
6004         #endif
6005         store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
6006         load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
6007         if(internal)
6008           assem_debug("branch: internal");
6009         else
6010           assem_debug("branch: external");
6011         if(internal&&is_ds[(ba[i]-start)>>2]) {
6012           ds_assemble_entry(i);
6013         }
6014         else {
6015           add_to_linker((int)out,ba[i],internal);
6016           emit_jmp(0);
6017         }
6018         set_jump_target(nottaken,(int)out);
6019       }
6020
6021       if(adj) {
6022         if(!invert) emit_addimm(cc,CLOCK_DIVIDER*adj,cc);
6023       }
6024     } // (!unconditional)
6025   } // if(ooo)
6026   else
6027   {
6028     // In-order execution (branch first)
6029     //DebugMessage(M64MSG_VERBOSE, "IOE");
6030     int nottaken=0;
6031     if(1) {
6032       //DebugMessage(M64MSG_VERBOSE, "branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
6033       if(1) {
6034         assert(fs>=0);
6035         emit_testimm(fs,0x800000);
6036         if(source[i]&0x10000) // BC1T
6037         {
6038           nottaken=(int)out;
6039           emit_jeq(1);
6040         }
6041         else // BC1F
6042         {
6043           nottaken=(int)out;
6044           emit_jne(1);
6045         }
6046       }
6047     } // if(!unconditional)
6048     int adj;
6049     uint64_t ds_unneeded=branch_regs[i].u;
6050     uint64_t ds_unneeded_upper=branch_regs[i].uu;
6051     ds_unneeded&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
6052     ds_unneeded_upper&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
6053     if((~ds_unneeded_upper>>rt1[i+1])&1) ds_unneeded_upper&=~((1LL<<dep1[i+1])|(1LL<<dep2[i+1]));
6054     ds_unneeded|=1;
6055     ds_unneeded_upper|=1;
6056     // branch taken
6057     //assem_debug("1:");
6058     wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
6059                   ds_unneeded,ds_unneeded_upper);
6060     // load regs
6061     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
6062     address_generation(i+1,&branch_regs[i],0);
6063     load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,INVCP);
6064     ds_assemble(i+1,&branch_regs[i]);
6065     cc=get_reg(branch_regs[i].regmap,CCREG);
6066     if(cc==-1) {
6067       emit_loadreg(CCREG,cc=HOST_CCREG);
6068       // CHECK: Is the following instruction (fall thru) allocated ok?
6069     }
6070     assert(cc==HOST_CCREG);
6071     store_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
6072     do_cc(i,i_regmap,&adj,ba[i],TAKEN,0);
6073     assem_debug("cycle count (adj)");
6074     if(adj) emit_addimm(cc,CLOCK_DIVIDER*(ccadj[i]+2-adj),cc);
6075     load_regs_bt(branch_regs[i].regmap,branch_regs[i].is32,branch_regs[i].dirty,ba[i]);
6076     if(internal)
6077       assem_debug("branch: internal");
6078     else
6079       assem_debug("branch: external");
6080     if(internal&&is_ds[(ba[i]-start)>>2]) {
6081       ds_assemble_entry(i);
6082     }
6083     else {
6084       add_to_linker((int)out,ba[i],internal);
6085       emit_jmp(0);
6086     }
6087
6088     // branch not taken
6089     if(1) { // <- FIXME (don't need this)
6090       set_jump_target(nottaken,(int)out);
6091       assem_debug("1:");
6092       if(!likely[i]) {
6093         wb_invalidate(regs[i].regmap,branch_regs[i].regmap,regs[i].dirty,regs[i].is32,
6094                       ds_unneeded,ds_unneeded_upper);
6095         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,rs1[i+1],rs2[i+1]);
6096         address_generation(i+1,&branch_regs[i],0);
6097         load_regs(regs[i].regmap,branch_regs[i].regmap,regs[i].was32,CCREG,CCREG);
6098         ds_assemble(i+1,&branch_regs[i]);
6099       }
6100       cc=get_reg(branch_regs[i].regmap,CCREG);
6101       if(cc==-1&&!likely[i]) {
6102         // Cycle count isn't in a register, temporarily load it then write it out
6103         emit_loadreg(CCREG,HOST_CCREG);
6104         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),HOST_CCREG);
6105         int jaddr=(int)out;
6106         emit_jns(0);
6107         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,NOTTAKEN,0);
6108         emit_storereg(CCREG,HOST_CCREG);
6109       }
6110       else{
6111         cc=get_reg(i_regmap,CCREG);
6112         assert(cc==HOST_CCREG);
6113         emit_addimm_and_set_flags(CLOCK_DIVIDER*(ccadj[i]+2),cc);
6114         int jaddr=(int)out;
6115         emit_jns(0);
6116         add_stub(CC_STUB,jaddr,(int)out,0,i,start+i*4+8,likely[i]?NULLDS:NOTTAKEN,0);
6117       }
6118     }
6119   }
6120 }
6121
6122 static void pagespan_assemble(int i,struct regstat *i_regs)
6123 {
6124   int s1l=get_reg(i_regs->regmap,rs1[i]);
6125   int s1h=get_reg(i_regs->regmap,rs1[i]|64);
6126   int s2l=get_reg(i_regs->regmap,rs2[i]);
6127   int s2h=get_reg(i_regs->regmap,rs2[i]|64);
6128   int taken=0;
6129   int nottaken=0;
6130   int unconditional=0;
6131   if(rs1[i]==0)
6132   {
6133     s1l=s2l;s1h=s2h;
6134     s2l=s2h=-1;
6135   }
6136   else if(rs2[i]==0)
6137   {
6138     s2l=s2h=-1;
6139   }
6140   if((i_regs->is32>>rs1[i])&(i_regs->is32>>rs2[i])&1) {
6141     s1h=s2h=-1;
6142   }
6143   int hr=0;
6144   int addr,alt,ntaddr;
6145   if(i_regs->regmap[HOST_BTREG]<0) {addr=HOST_BTREG;}
6146   else {
6147     while(hr<HOST_REGS)
6148     {
6149       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG &&
6150          (i_regs->regmap[hr]&63)!=rs1[i] &&
6151          (i_regs->regmap[hr]&63)!=rs2[i] )
6152       {
6153         addr=hr++;break;
6154       }
6155       hr++;
6156     }
6157   }
6158   while(hr<HOST_REGS)
6159   {
6160     if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
6161        (i_regs->regmap[hr]&63)!=rs1[i] &&
6162        (i_regs->regmap[hr]&63)!=rs2[i] )
6163     {
6164       alt=hr++;break;
6165     }
6166     hr++;
6167   }
6168   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ needs another register
6169   {
6170     while(hr<HOST_REGS)
6171     {
6172       if(hr!=EXCLUDE_REG && hr!=HOST_CCREG && hr!=HOST_BTREG &&
6173          (i_regs->regmap[hr]&63)!=rs1[i] &&
6174          (i_regs->regmap[hr]&63)!=rs2[i] )
6175       {
6176         ntaddr=hr;break;
6177       }
6178       hr++;
6179     }
6180   }
6181   assert(hr<HOST_REGS);
6182   if((opcode[i]&0x2e)==4||opcode[i]==0x11) { // BEQ/BNE/BEQL/BNEL/BC1
6183     load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,CCREG,CCREG);
6184   }
6185   emit_addimm(HOST_CCREG,CLOCK_DIVIDER*(ccadj[i]+2),HOST_CCREG);
6186   if(opcode[i]==2) // J
6187   {
6188     unconditional=1;
6189   }
6190   if(opcode[i]==3) // JAL
6191   {
6192     // TODO: mini_ht
6193     int rt=get_reg(i_regs->regmap,31);
6194     emit_movimm(start+i*4+8,rt);
6195     unconditional=1;
6196   }
6197   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
6198   {
6199     emit_mov(s1l,addr);
6200     if(opcode2[i]==9) // JALR
6201     {
6202       int rt=get_reg(i_regs->regmap,rt1[i]);
6203       emit_movimm(start+i*4+8,rt);
6204     }
6205   }
6206   if((opcode[i]&0x3f)==4) // BEQ
6207   {
6208     if(rs1[i]==rs2[i])
6209     {
6210       unconditional=1;
6211     }
6212     else
6213     #ifdef HAVE_CMOV_IMM
6214     if(s1h<0) {
6215       if(s2l>=0) emit_cmp(s1l,s2l);
6216       else emit_test(s1l,s1l);
6217       emit_cmov2imm_e_ne_compact(ba[i],start+i*4+8,addr);
6218     }
6219     else
6220     #endif
6221     {
6222       assert(s1l>=0);
6223       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
6224       if(s1h>=0) {
6225         if(s2h>=0) emit_cmp(s1h,s2h);
6226         else emit_test(s1h,s1h);
6227         emit_cmovne_reg(alt,addr);
6228       }
6229       if(s2l>=0) emit_cmp(s1l,s2l);
6230       else emit_test(s1l,s1l);
6231       emit_cmovne_reg(alt,addr);
6232     }
6233   }
6234   if((opcode[i]&0x3f)==5) // BNE
6235   {
6236     #ifdef HAVE_CMOV_IMM
6237     if(s1h<0) {
6238       if(s2l>=0) emit_cmp(s1l,s2l);
6239       else emit_test(s1l,s1l);
6240       emit_cmov2imm_e_ne_compact(start+i*4+8,ba[i],addr);
6241     }
6242     else
6243     #endif
6244     {
6245       assert(s1l>=0);
6246       emit_mov2imm_compact(start+i*4+8,addr,ba[i],alt);
6247       if(s1h>=0) {
6248         if(s2h>=0) emit_cmp(s1h,s2h);
6249         else emit_test(s1h,s1h);
6250         emit_cmovne_reg(alt,addr);
6251       }
6252       if(s2l>=0) emit_cmp(s1l,s2l);
6253       else emit_test(s1l,s1l);
6254       emit_cmovne_reg(alt,addr);
6255     }
6256   }
6257   if((opcode[i]&0x3f)==0x14) // BEQL
6258   {
6259     if(s1h>=0) {
6260       if(s2h>=0) emit_cmp(s1h,s2h);
6261       else emit_test(s1h,s1h);
6262       nottaken=(int)out;
6263       emit_jne(0);
6264     }
6265     if(s2l>=0) emit_cmp(s1l,s2l);
6266     else emit_test(s1l,s1l);
6267     if(nottaken) set_jump_target(nottaken,(int)out);
6268     nottaken=(int)out;
6269     emit_jne(0);
6270   }
6271   if((opcode[i]&0x3f)==0x15) // BNEL
6272   {
6273     if(s1h>=0) {
6274       if(s2h>=0) emit_cmp(s1h,s2h);
6275       else emit_test(s1h,s1h);
6276       taken=(int)out;
6277       emit_jne(0);
6278     }
6279     if(s2l>=0) emit_cmp(s1l,s2l);
6280     else emit_test(s1l,s1l);
6281     nottaken=(int)out;
6282     emit_jeq(0);
6283     if(taken) set_jump_target(taken,(int)out);
6284   }
6285   if((opcode[i]&0x3f)==6) // BLEZ
6286   {
6287     emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
6288     emit_cmpimm(s1l,1);
6289     if(s1h>=0) emit_mov(addr,ntaddr);
6290     emit_cmovl_reg(alt,addr);
6291     if(s1h>=0) {
6292       emit_test(s1h,s1h);
6293       emit_cmovne_reg(ntaddr,addr);
6294       emit_cmovs_reg(alt,addr);
6295     }
6296   }
6297   if((opcode[i]&0x3f)==7) // BGTZ
6298   {
6299     emit_mov2imm_compact(ba[i],addr,start+i*4+8,ntaddr);
6300     emit_cmpimm(s1l,1);
6301     if(s1h>=0) emit_mov(addr,alt);
6302     emit_cmovl_reg(ntaddr,addr);
6303     if(s1h>=0) {
6304       emit_test(s1h,s1h);
6305       emit_cmovne_reg(alt,addr);
6306       emit_cmovs_reg(ntaddr,addr);
6307     }
6308   }
6309   if((opcode[i]&0x3f)==0x16) // BLEZL
6310   {
6311     assert((opcode[i]&0x3f)!=0x16);
6312   }
6313   if((opcode[i]&0x3f)==0x17) // BGTZL
6314   {
6315     assert((opcode[i]&0x3f)!=0x17);
6316   }
6317   assert(opcode[i]!=1); // BLTZ/BGEZ
6318
6319   //FIXME: Check CSREG
6320   if(opcode[i]==0x11 && opcode2[i]==0x08 ) {
6321     if((source[i]&0x30000)==0) // BC1F
6322     {
6323       emit_mov2imm_compact(ba[i],addr,start+i*4+8,alt);
6324       emit_testimm(s1l,0x800000);
6325       emit_cmovne_reg(alt,addr);
6326     }
6327     if((source[i]&0x30000)==0x10000) // BC1T
6328     {
6329       emit_mov2imm_compact(ba[i],alt,start+i*4+8,addr);
6330       emit_testimm(s1l,0x800000);
6331       emit_cmovne_reg(alt,addr);
6332     }
6333     if((source[i]&0x30000)==0x20000) // BC1FL
6334     {
6335       emit_testimm(s1l,0x800000);
6336       nottaken=(int)out;
6337       emit_jne(0);
6338     }
6339     if((source[i]&0x30000)==0x30000) // BC1TL
6340     {
6341       emit_testimm(s1l,0x800000);
6342       nottaken=(int)out;
6343       emit_jeq(0);
6344     }
6345   }
6346
6347   assert(i_regs->regmap[HOST_CCREG]==CCREG);
6348   wb_dirtys(regs[i].regmap,regs[i].is32,regs[i].dirty);
6349   if(likely[i]||unconditional)
6350   {
6351     emit_movimm(ba[i],HOST_BTREG);
6352   }
6353   else if(addr!=HOST_BTREG)
6354   {
6355     emit_mov(addr,HOST_BTREG);
6356   }
6357   void *branch_addr=out;
6358   emit_jmp(0);
6359   int target_addr=start+i*4+5;
6360   void *stub=out;
6361   void *compiled_target_addr=check_addr(target_addr);
6362   emit_extjump_ds((int)branch_addr,target_addr);
6363   if(compiled_target_addr) {
6364     set_jump_target((int)branch_addr,(int)compiled_target_addr);
6365     add_link(target_addr,stub);
6366   }
6367   else set_jump_target((int)branch_addr,(int)stub);
6368   if(likely[i]) {
6369     // Not-taken path
6370     set_jump_target((int)nottaken,(int)out);
6371     wb_dirtys(regs[i].regmap,regs[i].is32,regs[i].dirty);
6372     void *branch_addr=out;
6373     emit_jmp(0);
6374     int target_addr=start+i*4+8;
6375     void *stub=out;
6376     void *compiled_target_addr=check_addr(target_addr);
6377     emit_extjump_ds((int)branch_addr,target_addr);
6378     if(compiled_target_addr) {
6379       set_jump_target((int)branch_addr,(int)compiled_target_addr);
6380       add_link(target_addr,stub);
6381     }
6382     else set_jump_target((int)branch_addr,(int)stub);
6383   }
6384 }
6385
6386 // Assemble the delay slot for the above
6387 static void pagespan_ds()
6388 {
6389   assem_debug("initial delay slot:");
6390   u_int vaddr=start+1;
6391   u_int page=(0x80000000^vaddr)>>12;
6392   u_int vpage=page;
6393   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[page^0x80000]^0x80000000)>>12;
6394   if(page>2048) page=2048+(page&2047);
6395   if(vpage>262143&&tlb_LUT_r[vaddr>>12]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
6396   if(vpage>2048) vpage=2048+(vpage&2047);
6397   ll_add(jump_dirty+vpage,vaddr,(void *)out);
6398   do_dirty_stub_ds();
6399   ll_add(jump_in+page,vaddr,(void *)out);
6400   assert(regs[0].regmap_entry[HOST_CCREG]==CCREG);
6401   if(regs[0].regmap[HOST_CCREG]!=CCREG)
6402     wb_register(CCREG,regs[0].regmap_entry,regs[0].wasdirty,regs[0].was32);
6403   if(regs[0].regmap[HOST_BTREG]!=BTREG)
6404     emit_writeword(HOST_BTREG,(int)&branch_target);
6405   load_regs(regs[0].regmap_entry,regs[0].regmap,regs[0].was32,rs1[0],rs2[0]);
6406   address_generation(0,&regs[0],regs[0].regmap_entry);
6407   if(itype[0]==LOAD||itype[0]==LOADLR||itype[0]==STORE||itype[0]==STORELR||itype[0]==C1LS)
6408     load_regs(regs[0].regmap_entry,regs[0].regmap,regs[0].was32,MMREG,ROREG);
6409   if(itype[0]==STORE||itype[0]==STORELR||(opcode[0]&0x3b)==0x39)
6410     load_regs(regs[0].regmap_entry,regs[0].regmap,regs[0].was32,INVCP,INVCP);
6411   cop1_usable=0;
6412   is_delayslot=0;
6413   switch(itype[0]) {
6414     case ALU:
6415       alu_assemble(0,&regs[0]);break;
6416     case IMM16:
6417       imm16_assemble(0,&regs[0]);break;
6418     case SHIFT:
6419       shift_assemble(0,&regs[0]);break;
6420     case SHIFTIMM:
6421       shiftimm_assemble(0,&regs[0]);break;
6422     case LOAD:
6423       load_assemble(0,&regs[0]);break;
6424     case LOADLR:
6425       loadlr_assemble(0,&regs[0]);break;
6426     case STORE:
6427       store_assemble(0,&regs[0]);break;
6428     case STORELR:
6429       storelr_assemble(0,&regs[0]);break;
6430     case COP0:
6431       cop0_assemble(0,&regs[0]);break;
6432     case COP1:
6433       cop1_assemble(0,&regs[0]);break;
6434     case C1LS:
6435       c1ls_assemble(0,&regs[0]);break;
6436     case FCONV:
6437       fconv_assemble(0,&regs[0]);break;
6438     case FLOAT:
6439       float_assemble(0,&regs[0]);break;
6440     case FCOMP:
6441       fcomp_assemble(0,&regs[0]);break;
6442     case MULTDIV:
6443       multdiv_assemble(0,&regs[0]);break;
6444     case MOV:
6445       mov_assemble(0,&regs[0]);break;
6446     case SYSCALL:
6447     case SPAN:
6448     case UJUMP:
6449     case RJUMP:
6450     case CJUMP:
6451     case SJUMP:
6452     case FJUMP:
6453       DebugMessage(M64MSG_VERBOSE, "Jump in the delay slot.  This is probably a bug.");
6454   }
6455   int btaddr=get_reg(regs[0].regmap,BTREG);
6456   if(btaddr<0) {
6457     btaddr=get_reg(regs[0].regmap,-1);
6458     emit_readword((int)&branch_target,btaddr);
6459   }
6460   assert(btaddr!=HOST_CCREG);
6461   if(regs[0].regmap[HOST_CCREG]!=CCREG) emit_loadreg(CCREG,HOST_CCREG);
6462 #ifdef HOST_IMM8
6463   emit_movimm(start+4,HOST_TEMPREG);
6464   emit_cmp(btaddr,HOST_TEMPREG);
6465 #else
6466   emit_cmpimm(btaddr,start+4);
6467 #endif
6468   int branch=(int)out;
6469   emit_jeq(0);
6470   store_regs_bt(regs[0].regmap,regs[0].is32,regs[0].dirty,-1);
6471   emit_jmp(jump_vaddr_reg[btaddr]);
6472   set_jump_target(branch,(int)out);
6473   store_regs_bt(regs[0].regmap,regs[0].is32,regs[0].dirty,start+4);
6474   load_regs_bt(regs[0].regmap,regs[0].is32,regs[0].dirty,start+4);
6475 }
6476
6477 // Basic liveness analysis for MIPS registers
6478 static void unneeded_registers(int istart,int iend,int r)
6479 {
6480   int i;
6481   uint64_t u,uu,b,bu;
6482   uint64_t temp_u,temp_uu;
6483   uint64_t tdep;
6484   if(iend==slen-1) {
6485     u=1;uu=1;
6486   }else{
6487     u=unneeded_reg[iend+1];
6488     uu=unneeded_reg_upper[iend+1];
6489     u=1;uu=1;
6490   }
6491   for (i=iend;i>=istart;i--)
6492   {
6493     //DebugMessage(M64MSG_VERBOSE, "unneeded registers i=%d (%d,%d) r=%d",i,istart,iend,r);
6494     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
6495     {
6496       // If subroutine call, flag return address as a possible branch target
6497       if(rt1[i]==31 && i<slen-2) bt[i+2]=1;
6498
6499       if(ba[i]<start || ba[i]>=(start+slen*4))
6500       {
6501         // Branch out of this block, flush all regs
6502         u=1;
6503         uu=1;
6504         /* Hexagon hack
6505         if(itype[i]==UJUMP&&rt1[i]==31)
6506         {
6507           uu=u=0x300C00F; // Discard at, v0-v1, t6-t9
6508         }
6509         if(itype[i]==RJUMP&&rs1[i]==31)
6510         {
6511           uu=u=0x300C0F3; // Discard at, a0-a3, t6-t9
6512         }
6513         if(start>0x80000400&&start<0x80800000) {
6514           if(itype[i]==UJUMP&&rt1[i]==31)
6515           {
6516             //uu=u=0x30300FF0FLL; // Discard at, v0-v1, t0-t9, lo, hi
6517             uu=u=0x300FF0F; // Discard at, v0-v1, t0-t9
6518           }
6519           if(itype[i]==RJUMP&&rs1[i]==31)
6520           {
6521             //uu=u=0x30300FFF3LL; // Discard at, a0-a3, t0-t9, lo, hi
6522             uu=u=0x300FFF3; // Discard at, a0-a3, t0-t9
6523           }
6524         }*/
6525         branch_unneeded_reg[i]=u;
6526         branch_unneeded_reg_upper[i]=uu;
6527         // Merge in delay slot
6528         tdep=(~uu>>rt1[i+1])&1;
6529         u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6530         uu|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6531         u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
6532         uu&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
6533         uu&=~((tdep<<dep1[i+1])|(tdep<<dep2[i+1]));
6534         u|=1;uu|=1;
6535         // If branch is "likely" (and conditional)
6536         // then we skip the delay slot on the fall-thru path
6537         if(likely[i]) {
6538           if(i<slen-1) {
6539             u&=unneeded_reg[i+2];
6540             uu&=unneeded_reg_upper[i+2];
6541           }
6542           else
6543           {
6544             u=1;
6545             uu=1;
6546           }
6547         }
6548       }
6549       else
6550       {
6551         // Internal branch, flag target
6552         bt[(ba[i]-start)>>2]=1;
6553         if(ba[i]<=start+i*4) {
6554           // Backward branch
6555           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
6556           {
6557             // Unconditional branch
6558             temp_u=1;temp_uu=1;
6559           } else {
6560             // Conditional branch (not taken case)
6561             temp_u=unneeded_reg[i+2];
6562             temp_uu=unneeded_reg_upper[i+2];
6563           }
6564           // Merge in delay slot
6565           tdep=(~temp_uu>>rt1[i+1])&1;
6566           temp_u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6567           temp_uu|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6568           temp_u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
6569           temp_uu&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
6570           temp_uu&=~((tdep<<dep1[i+1])|(tdep<<dep2[i+1]));
6571           temp_u|=1;temp_uu|=1;
6572           // If branch is "likely" (and conditional)
6573           // then we skip the delay slot on the fall-thru path
6574           if(likely[i]) {
6575             if(i<slen-1) {
6576               temp_u&=unneeded_reg[i+2];
6577               temp_uu&=unneeded_reg_upper[i+2];
6578             }
6579             else
6580             {
6581               temp_u=1;
6582               temp_uu=1;
6583             }
6584           }
6585           tdep=(~temp_uu>>rt1[i])&1;
6586           temp_u|=(1LL<<rt1[i])|(1LL<<rt2[i]);
6587           temp_uu|=(1LL<<rt1[i])|(1LL<<rt2[i]);
6588           temp_u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
6589           temp_uu&=~((1LL<<us1[i])|(1LL<<us2[i]));
6590           temp_uu&=~((tdep<<dep1[i])|(tdep<<dep2[i]));
6591           temp_u|=1;temp_uu|=1;
6592           unneeded_reg[i]=temp_u;
6593           unneeded_reg_upper[i]=temp_uu;
6594           // Only go three levels deep.  This recursion can take an
6595           // excessive amount of time if there are a lot of nested loops.
6596           if(r<2) {
6597             unneeded_registers((ba[i]-start)>>2,i-1,r+1);
6598           }else{
6599             unneeded_reg[(ba[i]-start)>>2]=1;
6600             unneeded_reg_upper[(ba[i]-start)>>2]=1;
6601           }
6602         } /*else*/ if(1) {
6603           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
6604           {
6605             // Unconditional branch
6606             u=unneeded_reg[(ba[i]-start)>>2];
6607             uu=unneeded_reg_upper[(ba[i]-start)>>2];
6608             branch_unneeded_reg[i]=u;
6609             branch_unneeded_reg_upper[i]=uu;
6610         //u=1;
6611         //uu=1;
6612         //branch_unneeded_reg[i]=u;
6613         //branch_unneeded_reg_upper[i]=uu;
6614             // Merge in delay slot
6615             tdep=(~uu>>rt1[i+1])&1;
6616             u|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6617             uu|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6618             u&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
6619             uu&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
6620             uu&=~((tdep<<dep1[i+1])|(tdep<<dep2[i+1]));
6621             u|=1;uu|=1;
6622           } else {
6623             // Conditional branch
6624             b=unneeded_reg[(ba[i]-start)>>2];
6625             bu=unneeded_reg_upper[(ba[i]-start)>>2];
6626             branch_unneeded_reg[i]=b;
6627             branch_unneeded_reg_upper[i]=bu;
6628         //b=1;
6629         //bu=1;
6630         //branch_unneeded_reg[i]=b;
6631         //branch_unneeded_reg_upper[i]=bu;
6632             // Branch delay slot
6633             tdep=(~uu>>rt1[i+1])&1;
6634             b|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6635             bu|=(1LL<<rt1[i+1])|(1LL<<rt2[i+1]);
6636             b&=~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
6637             bu&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
6638             bu&=~((tdep<<dep1[i+1])|(tdep<<dep2[i+1]));
6639             b|=1;bu|=1;
6640             // If branch is "likely" then we skip the
6641             // delay slot on the fall-thru path
6642             if(likely[i]) {
6643               u=b;
6644               uu=bu;
6645               if(i<slen-1) {
6646                 u&=unneeded_reg[i+2];
6647                 uu&=unneeded_reg_upper[i+2];
6648         //u=1;
6649         //uu=1;
6650               }
6651             } else {
6652               u&=b;
6653               uu&=bu;
6654         //u=1;
6655         //uu=1;
6656             }
6657             if(i<slen-1) {
6658               branch_unneeded_reg[i]&=unneeded_reg[i+2];
6659               branch_unneeded_reg_upper[i]&=unneeded_reg_upper[i+2];
6660         //branch_unneeded_reg[i]=1;
6661         //branch_unneeded_reg_upper[i]=1;
6662             } else {
6663               branch_unneeded_reg[i]=1;
6664               branch_unneeded_reg_upper[i]=1;
6665             }
6666           }
6667         }
6668       }
6669     }
6670     else if(itype[i]==SYSCALL)
6671     {
6672       // SYSCALL instruction (software interrupt)
6673       u=1;
6674       uu=1;
6675     }
6676     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
6677     {
6678       // ERET instruction (return from interrupt)
6679       u=1;
6680       uu=1;
6681     }
6682     //u=uu=1; // DEBUG
6683     tdep=(~uu>>rt1[i])&1;
6684     // Written registers are unneeded
6685     u|=1LL<<rt1[i];
6686     u|=1LL<<rt2[i];
6687     uu|=1LL<<rt1[i];
6688     uu|=1LL<<rt2[i];
6689     // Accessed registers are needed
6690     u&=~(1LL<<rs1[i]);
6691     u&=~(1LL<<rs2[i]);
6692     uu&=~(1LL<<us1[i]);
6693     uu&=~(1LL<<us2[i]);
6694     // Source-target dependencies
6695     uu&=~(tdep<<dep1[i]);
6696     uu&=~(tdep<<dep2[i]);
6697     // R0 is always unneeded
6698     u|=1;uu|=1;
6699     // Save it
6700     unneeded_reg[i]=u;
6701     unneeded_reg_upper[i]=uu;
6702     /*
6703     DebugMessage(M64MSG_VERBOSE, "ur (%d,%d) %x: ",istart,iend,start+i*4);
6704     DebugMessage(M64MSG_VERBOSE, "U:");
6705     int r;
6706     for(r=1;r<=CCREG;r++) {
6707       if((unneeded_reg[i]>>r)&1) {
6708         if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
6709         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
6710         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
6711       }
6712     }
6713     DebugMessage(M64MSG_VERBOSE, " UU:");
6714     for(r=1;r<=CCREG;r++) {
6715       if(((unneeded_reg_upper[i]&~unneeded_reg[i])>>r)&1) {
6716         if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
6717         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
6718         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
6719       }
6720     }*/
6721   }
6722 }
6723
6724 // Identify registers which are likely to contain 32-bit values
6725 // This is used to predict whether any branches will jump to a
6726 // location with 64-bit values in registers.
6727 static void provisional_32bit()
6728 {
6729   int i,j;
6730   uint64_t is32=1;
6731   uint64_t lastbranch=1;
6732
6733   for(i=0;i<slen;i++)
6734   {
6735     if(i>0) {
6736       if(itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP) {
6737         if(i>1) is32=lastbranch;
6738         else is32=1;
6739       }
6740     }
6741     if(i>1)
6742     {
6743       if(itype[i-2]==CJUMP||itype[i-2]==SJUMP||itype[i-2]==FJUMP) {
6744         if(likely[i-2]) {
6745           if(i>2) is32=lastbranch;
6746           else is32=1;
6747         }
6748       }
6749       if((opcode[i-2]&0x2f)==0x05) // BNE/BNEL
6750       {
6751         if(rs1[i-2]==0||rs2[i-2]==0)
6752         {
6753           if(rs1[i-2]) {
6754             is32|=1LL<<rs1[i-2];
6755           }
6756           if(rs2[i-2]) {
6757             is32|=1LL<<rs2[i-2];
6758           }
6759         }
6760       }
6761     }
6762     // If something jumps here with 64-bit values
6763     // then promote those registers to 64 bits
6764     if(bt[i])
6765     {
6766       uint64_t temp_is32=is32;
6767       for(j=i-1;j>=0;j--)
6768       {
6769         if(ba[j]==start+i*4)
6770           //temp_is32&=branch_regs[j].is32;
6771           temp_is32&=p32[j];
6772       }
6773       for(j=i;j<slen;j++)
6774       {
6775         if(ba[j]==start+i*4)
6776           temp_is32=1;
6777       }
6778       is32=temp_is32;
6779     }
6780     int type=itype[i];
6781     int op=opcode[i];
6782     int op2=opcode2[i];
6783     int rt=rt1[i];
6784     int s1=rs1[i];
6785     int s2=rs2[i];
6786     if(type==UJUMP||type==RJUMP||type==CJUMP||type==SJUMP||type==FJUMP) {
6787       // Branches don't write registers, consider the delay slot instead.
6788       type=itype[i+1];
6789       op=opcode[i+1];
6790       op2=opcode2[i+1];
6791       rt=rt1[i+1];
6792       s1=rs1[i+1];
6793       s2=rs2[i+1];
6794       lastbranch=is32;
6795     }
6796     switch(type) {
6797       case LOAD:
6798         if(opcode[i]==0x27||opcode[i]==0x37|| // LWU/LD
6799            opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
6800           is32&=~(1LL<<rt);
6801         else
6802           is32|=1LL<<rt;
6803         break;
6804       case STORE:
6805       case STORELR:
6806         break;
6807       case LOADLR:
6808         if(op==0x1a||op==0x1b) is32&=~(1LL<<rt); // LDR/LDL
6809         if(op==0x22) is32|=1LL<<rt; // LWL
6810         break;
6811       case IMM16:
6812         if (op==0x08||op==0x09|| // ADDI/ADDIU
6813             op==0x0a||op==0x0b|| // SLTI/SLTIU
6814             op==0x0c|| // ANDI
6815             op==0x0f)  // LUI
6816         {
6817           is32|=1LL<<rt;
6818         }
6819         if(op==0x18||op==0x19) { // DADDI/DADDIU
6820           is32&=~(1LL<<rt);
6821           //if(imm[i]==0)
6822           //  is32|=((is32>>s1)&1LL)<<rt;
6823         }
6824         if(op==0x0d||op==0x0e) { // ORI/XORI
6825           uint64_t sr=((is32>>s1)&1LL);
6826           is32&=~(1LL<<rt);
6827           is32|=sr<<rt;
6828         }
6829         break;
6830       case UJUMP:
6831         break;
6832       case RJUMP:
6833         break;
6834       case CJUMP:
6835         break;
6836       case SJUMP:
6837         break;
6838       case FJUMP:
6839         break;
6840       case ALU:
6841         if(op2>=0x20&&op2<=0x23) { // ADD/ADDU/SUB/SUBU
6842           is32|=1LL<<rt;
6843         }
6844         if(op2==0x2a||op2==0x2b) { // SLT/SLTU
6845           is32|=1LL<<rt;
6846         }
6847         else if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
6848           uint64_t sr=((is32>>s1)&(is32>>s2)&1LL);
6849           is32&=~(1LL<<rt);
6850           is32|=sr<<rt;
6851         }
6852         else if(op2>=0x2c&&op2<=0x2d) { // DADD/DADDU
6853           if(s1==0&&s2==0) {
6854             is32|=1LL<<rt;
6855           }
6856           else if(s2==0) {
6857             uint64_t sr=((is32>>s1)&1LL);
6858             is32&=~(1LL<<rt);
6859             is32|=sr<<rt;
6860           }
6861           else if(s1==0) {
6862             uint64_t sr=((is32>>s2)&1LL);
6863             is32&=~(1LL<<rt);
6864             is32|=sr<<rt;
6865           }
6866           else {
6867             is32&=~(1LL<<rt);
6868           }
6869         }
6870         else if(op2>=0x2e&&op2<=0x2f) { // DSUB/DSUBU
6871           if(s1==0&&s2==0) {
6872             is32|=1LL<<rt;
6873           }
6874           else if(s2==0) {
6875             uint64_t sr=((is32>>s1)&1LL);
6876             is32&=~(1LL<<rt);
6877             is32|=sr<<rt;
6878           }
6879           else {
6880             is32&=~(1LL<<rt);
6881           }
6882         }
6883         break;
6884       case MULTDIV:
6885         if (op2>=0x1c&&op2<=0x1f) { // DMULT/DMULTU/DDIV/DDIVU
6886           is32&=~((1LL<<HIREG)|(1LL<<LOREG));
6887         }
6888         else {
6889           is32|=(1LL<<HIREG)|(1LL<<LOREG);
6890         }
6891         break;
6892       case MOV:
6893         {
6894           uint64_t sr=((is32>>s1)&1LL);
6895           is32&=~(1LL<<rt);
6896           is32|=sr<<rt;
6897         }
6898         break;
6899       case SHIFT:
6900         if(op2>=0x14&&op2<=0x17) is32&=~(1LL<<rt); // DSLLV/DSRLV/DSRAV
6901         else is32|=1LL<<rt; // SLLV/SRLV/SRAV
6902         break;
6903       case SHIFTIMM:
6904         is32|=1LL<<rt;
6905         // DSLL/DSRL/DSRA/DSLL32/DSRL32 but not DSRA32 have 64-bit result
6906         if(op2>=0x38&&op2<0x3f) is32&=~(1LL<<rt);
6907         break;
6908       case COP0:
6909         if(op2==0) is32|=1LL<<rt; // MFC0
6910         break;
6911       case COP1:
6912         if(op2==0) is32|=1LL<<rt; // MFC1
6913         if(op2==1) is32&=~(1LL<<rt); // DMFC1
6914         if(op2==2) is32|=1LL<<rt; // CFC1
6915         break;
6916       case C1LS:
6917         break;
6918       case FLOAT:
6919       case FCONV:
6920         break;
6921       case FCOMP:
6922         break;
6923       case SYSCALL:
6924         break;
6925       default:
6926         break;
6927     }
6928     is32|=1;
6929     p32[i]=is32;
6930
6931     if(i>0)
6932     {
6933       if(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)
6934       {
6935         if(rt1[i-1]==31) // JAL/JALR
6936         {
6937           // Subroutine call will return here, don't alloc any registers
6938           is32=1;
6939         }
6940         else if(i+1<slen)
6941         {
6942           // Internal branch will jump here, match registers to caller
6943           is32=0x3FFFFFFFFLL;
6944         }
6945       }
6946     }
6947   }
6948 }
6949
6950 // Identify registers which may be assumed to contain 32-bit values
6951 // and where optimizations will rely on this.
6952 // This is used to determine whether backward branches can safely
6953 // jump to a location with 64-bit values in registers.
6954 static void provisional_r32()
6955 {
6956   u_int r32=0;
6957   int i;
6958
6959   for (i=slen-1;i>=0;i--)
6960   {
6961     int hr;
6962     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
6963     {
6964       if(ba[i]<start || ba[i]>=(start+slen*4))
6965       {
6966         // Branch out of this block, don't need anything
6967         r32=0;
6968       }
6969       else
6970       {
6971         // Internal branch
6972         // Need whatever matches the target
6973         // (and doesn't get overwritten by the delay slot instruction)
6974         r32=0;
6975         int t=(ba[i]-start)>>2;
6976         if(ba[i]>start+i*4) {
6977           // Forward branch
6978           //if(!(requires_32bit[t]&~regs[i].was32))
6979           //  r32|=requires_32bit[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
6980           if(!(pr32[t]&~regs[i].was32))
6981             r32|=pr32[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
6982         }else{
6983           // Backward branch
6984           if(!(regs[t].was32&~unneeded_reg_upper[t]&~regs[i].was32))
6985             r32|=regs[t].was32&~unneeded_reg_upper[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
6986         }
6987       }
6988       // Conditional branch may need registers for following instructions
6989       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
6990       {
6991         if(i<slen-2) {
6992           //r32|=requires_32bit[i+2];
6993           r32|=pr32[i+2];
6994           r32&=regs[i].was32;
6995           // Mark this address as a branch target since it may be called
6996           // upon return from interrupt
6997           //bt[i+2]=1;
6998         }
6999       }
7000       // Merge in delay slot
7001       if(!likely[i]) {
7002         // These are overwritten unless the branch is "likely"
7003         // and the delay slot is nullified if not taken
7004         r32&=~(1LL<<rt1[i+1]);
7005         r32&=~(1LL<<rt2[i+1]);
7006       }
7007       // Assume these are needed (delay slot)
7008       if(us1[i+1]>0)
7009       {
7010         if((regs[i].was32>>us1[i+1])&1) r32|=1LL<<us1[i+1];
7011       }
7012       if(us2[i+1]>0)
7013       {
7014         if((regs[i].was32>>us2[i+1])&1) r32|=1LL<<us2[i+1];
7015       }
7016       if(dep1[i+1]&&!((unneeded_reg_upper[i]>>dep1[i+1])&1))
7017       {
7018         if((regs[i].was32>>dep1[i+1])&1) r32|=1LL<<dep1[i+1];
7019       }
7020       if(dep2[i+1]&&!((unneeded_reg_upper[i]>>dep2[i+1])&1))
7021       {
7022         if((regs[i].was32>>dep2[i+1])&1) r32|=1LL<<dep2[i+1];
7023       }
7024     }
7025     else if(itype[i]==SYSCALL)
7026     {
7027       // SYSCALL instruction (software interrupt)
7028       r32=0;
7029     }
7030     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
7031     {
7032       // ERET instruction (return from interrupt)
7033       r32=0;
7034     }
7035     // Check 32 bits
7036     r32&=~(1LL<<rt1[i]);
7037     r32&=~(1LL<<rt2[i]);
7038     if(us1[i]>0)
7039     {
7040       if((regs[i].was32>>us1[i])&1) r32|=1LL<<us1[i];
7041     }
7042     if(us2[i]>0)
7043     {
7044       if((regs[i].was32>>us2[i])&1) r32|=1LL<<us2[i];
7045     }
7046     if(dep1[i]&&!((unneeded_reg_upper[i]>>dep1[i])&1))
7047     {
7048       if((regs[i].was32>>dep1[i])&1) r32|=1LL<<dep1[i];
7049     }
7050     if(dep2[i]&&!((unneeded_reg_upper[i]>>dep2[i])&1))
7051     {
7052       if((regs[i].was32>>dep2[i])&1) r32|=1LL<<dep2[i];
7053     }
7054     //requires_32bit[i]=r32;
7055     pr32[i]=r32;
7056
7057     // Dirty registers which are 32-bit, require 32-bit input
7058     // as they will be written as 32-bit values
7059     for(hr=0;hr<HOST_REGS;hr++)
7060     {
7061       if(regs[i].regmap_entry[hr]>0&&regs[i].regmap_entry[hr]<64) {
7062         if((regs[i].was32>>regs[i].regmap_entry[hr])&(regs[i].wasdirty>>hr)&1) {
7063           if(!((unneeded_reg_upper[i]>>regs[i].regmap_entry[hr])&1))
7064           pr32[i]|=1LL<<regs[i].regmap_entry[hr];
7065           //requires_32bit[i]|=1LL<<regs[i].regmap_entry[hr];
7066         }
7067       }
7068     }
7069   }
7070 }
7071
7072 // Write back dirty registers as soon as we will no longer modify them,
7073 // so that we don't end up with lots of writes at the branches.
7074 static void clean_registers(int istart,int iend,int wr)
7075 {
7076   int i;
7077   int r;
7078   u_int will_dirty_i,will_dirty_next,temp_will_dirty;
7079   u_int wont_dirty_i,wont_dirty_next,temp_wont_dirty;
7080   if(iend==slen-1) {
7081     will_dirty_i=will_dirty_next=0;
7082     wont_dirty_i=wont_dirty_next=0;
7083   }else{
7084     will_dirty_i=will_dirty_next=will_dirty[iend+1];
7085     wont_dirty_i=wont_dirty_next=wont_dirty[iend+1];
7086   }
7087   for (i=iend;i>=istart;i--)
7088   {
7089     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
7090     {
7091       if(ba[i]<start || ba[i]>=(start+slen*4))
7092       {
7093         // Branch out of this block, flush all regs
7094         if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
7095         {
7096           // Unconditional branch
7097           will_dirty_i=0;
7098           wont_dirty_i=0;
7099           // Merge in delay slot (will dirty)
7100           for(r=0;r<HOST_REGS;r++) {
7101             if(r!=EXCLUDE_REG) {
7102               if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7103               if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7104               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7105               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7106               if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7107               if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7108               if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7109               if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7110               if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7111               if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7112               if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7113               if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7114               if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7115               if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7116             }
7117           }
7118         }
7119         else
7120         {
7121           // Conditional branch
7122           will_dirty_i=0;
7123           wont_dirty_i=wont_dirty_next;
7124           // Merge in delay slot (will dirty)
7125           for(r=0;r<HOST_REGS;r++) {
7126             if(r!=EXCLUDE_REG) {
7127               if(!likely[i]) {
7128                 // Might not dirty if likely branch is not taken
7129                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7130                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7131                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7132                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7133                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7134                 if(branch_regs[i].regmap[r]==0) will_dirty_i&=~(1<<r);
7135                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7136                 //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7137                 //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7138                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7139                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7140                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7141                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7142                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7143               }
7144             }
7145           }
7146         }
7147         // Merge in delay slot (wont dirty)
7148         for(r=0;r<HOST_REGS;r++) {
7149           if(r!=EXCLUDE_REG) {
7150             if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
7151             if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
7152             if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
7153             if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
7154             if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
7155             if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
7156             if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
7157             if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
7158             if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
7159             if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
7160           }
7161         }
7162         if(wr) {
7163           #ifndef DESTRUCTIVE_WRITEBACK
7164           branch_regs[i].dirty&=wont_dirty_i;
7165           #endif
7166           branch_regs[i].dirty|=will_dirty_i;
7167         }
7168       }
7169       else
7170       {
7171         // Internal branch
7172         if(ba[i]<=start+i*4) {
7173           // Backward branch
7174           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
7175           {
7176             // Unconditional branch
7177             temp_will_dirty=0;
7178             temp_wont_dirty=0;
7179             // Merge in delay slot (will dirty)
7180             for(r=0;r<HOST_REGS;r++) {
7181               if(r!=EXCLUDE_REG) {
7182                 if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
7183                 if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
7184                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
7185                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
7186                 if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
7187                 if(branch_regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
7188                 if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
7189                 if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
7190                 if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
7191                 if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
7192                 if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
7193                 if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
7194                 if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
7195                 if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
7196               }
7197             }
7198           } else {
7199             // Conditional branch (not taken case)
7200             temp_will_dirty=will_dirty_next;
7201             temp_wont_dirty=wont_dirty_next;
7202             // Merge in delay slot (will dirty)
7203             for(r=0;r<HOST_REGS;r++) {
7204               if(r!=EXCLUDE_REG) {
7205                 if(!likely[i]) {
7206                   // Will not dirty if likely branch is not taken
7207                   if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
7208                   if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
7209                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
7210                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
7211                   if((branch_regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
7212                   if(branch_regs[i].regmap[r]==0) temp_will_dirty&=~(1<<r);
7213                   if(branch_regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
7214                   //if((regs[i].regmap[r]&63)==rt1[i]) temp_will_dirty|=1<<r;
7215                   //if((regs[i].regmap[r]&63)==rt2[i]) temp_will_dirty|=1<<r;
7216                   if((regs[i].regmap[r]&63)==rt1[i+1]) temp_will_dirty|=1<<r;
7217                   if((regs[i].regmap[r]&63)==rt2[i+1]) temp_will_dirty|=1<<r;
7218                   if((regs[i].regmap[r]&63)>33) temp_will_dirty&=~(1<<r);
7219                   if(regs[i].regmap[r]<=0) temp_will_dirty&=~(1<<r);
7220                   if(regs[i].regmap[r]==CCREG) temp_will_dirty|=1<<r;
7221                 }
7222               }
7223             }
7224           }
7225           // Merge in delay slot (wont dirty)
7226           for(r=0;r<HOST_REGS;r++) {
7227             if(r!=EXCLUDE_REG) {
7228               if((regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
7229               if((regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
7230               if((regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
7231               if((regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
7232               if(regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
7233               if((branch_regs[i].regmap[r]&63)==rt1[i]) temp_wont_dirty|=1<<r;
7234               if((branch_regs[i].regmap[r]&63)==rt2[i]) temp_wont_dirty|=1<<r;
7235               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) temp_wont_dirty|=1<<r;
7236               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) temp_wont_dirty|=1<<r;
7237               if(branch_regs[i].regmap[r]==CCREG) temp_wont_dirty|=1<<r;
7238             }
7239           }
7240           // Deal with changed mappings
7241           if(i<iend) {
7242             for(r=0;r<HOST_REGS;r++) {
7243               if(r!=EXCLUDE_REG) {
7244                 if(regs[i].regmap[r]!=regmap_pre[i][r]) {
7245                   temp_will_dirty&=~(1<<r);
7246                   temp_wont_dirty&=~(1<<r);
7247                   if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
7248                     temp_will_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
7249                     temp_wont_dirty|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
7250                   } else {
7251                     temp_will_dirty|=1<<r;
7252                     temp_wont_dirty|=1<<r;
7253                   }
7254                 }
7255               }
7256             }
7257           }
7258           if(wr) {
7259             will_dirty[i]=temp_will_dirty;
7260             wont_dirty[i]=temp_wont_dirty;
7261             clean_registers((ba[i]-start)>>2,i-1,0);
7262           }else{
7263             // Limit recursion.  It can take an excessive amount
7264             // of time if there are a lot of nested loops.
7265             will_dirty[(ba[i]-start)>>2]=0;
7266             wont_dirty[(ba[i]-start)>>2]=-1;
7267           }
7268         }
7269         /*else*/ if(1)
7270         {
7271           if(itype[i]==RJUMP||itype[i]==UJUMP||(source[i]>>16)==0x1000)
7272           {
7273             // Unconditional branch
7274             will_dirty_i=0;
7275             wont_dirty_i=0;
7276           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
7277             for(r=0;r<HOST_REGS;r++) {
7278               if(r!=EXCLUDE_REG) {
7279                 if(branch_regs[i].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
7280                   will_dirty_i|=will_dirty[(ba[i]-start)>>2]&(1<<r);
7281                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
7282                 }
7283                 if(branch_regs[i].regmap[r]>=0) {
7284                   will_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
7285                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(branch_regs[i].regmap[r]&63))&1)<<r;
7286                 }
7287               }
7288             }
7289           //}
7290             // Merge in delay slot
7291             for(r=0;r<HOST_REGS;r++) {
7292               if(r!=EXCLUDE_REG) {
7293                 if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7294                 if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7295                 if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7296                 if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7297                 if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7298                 if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7299                 if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7300                 if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7301                 if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7302                 if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7303                 if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7304                 if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7305                 if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7306                 if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7307               }
7308             }
7309           } else {
7310             // Conditional branch
7311             will_dirty_i=will_dirty_next;
7312             wont_dirty_i=wont_dirty_next;
7313           //if(ba[i]>start+i*4) { // Disable recursion (for debugging)
7314             for(r=0;r<HOST_REGS;r++) {
7315               if(r!=EXCLUDE_REG) {
7316                 signed char target_reg=branch_regs[i].regmap[r];
7317                 if(target_reg==regs[(ba[i]-start)>>2].regmap_entry[r]) {
7318                   will_dirty_i&=will_dirty[(ba[i]-start)>>2]&(1<<r);
7319                   wont_dirty_i|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
7320                 }
7321                 else if(target_reg>=0) {
7322                   will_dirty_i&=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
7323                   wont_dirty_i|=((unneeded_reg[(ba[i]-start)>>2]>>(target_reg&63))&1)<<r;
7324                 }
7325                 // Treat delay slot as part of branch too
7326                 /*if(regs[i+1].regmap[r]==regs[(ba[i]-start)>>2].regmap_entry[r]) {
7327                   will_dirty[i+1]&=will_dirty[(ba[i]-start)>>2]&(1<<r);
7328                   wont_dirty[i+1]|=wont_dirty[(ba[i]-start)>>2]&(1<<r);
7329                 }
7330                 else
7331                 {
7332                   will_dirty[i+1]&=~(1<<r);
7333                 }*/
7334               }
7335             }
7336           //}
7337             // Merge in delay slot
7338             for(r=0;r<HOST_REGS;r++) {
7339               if(r!=EXCLUDE_REG) {
7340                 if(!likely[i]) {
7341                   // Might not dirty if likely branch is not taken
7342                   if((branch_regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7343                   if((branch_regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7344                   if((branch_regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7345                   if((branch_regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7346                   if((branch_regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7347                   if(branch_regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7348                   if(branch_regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7349                   //if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7350                   //if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7351                   if((regs[i].regmap[r]&63)==rt1[i+1]) will_dirty_i|=1<<r;
7352                   if((regs[i].regmap[r]&63)==rt2[i+1]) will_dirty_i|=1<<r;
7353                   if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7354                   if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7355                   if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7356                 }
7357               }
7358             }
7359           }
7360           // Merge in delay slot (won't dirty)
7361           for(r=0;r<HOST_REGS;r++) {
7362             if(r!=EXCLUDE_REG) {
7363               if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
7364               if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
7365               if((regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
7366               if((regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
7367               if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
7368               if((branch_regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
7369               if((branch_regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
7370               if((branch_regs[i].regmap[r]&63)==rt1[i+1]) wont_dirty_i|=1<<r;
7371               if((branch_regs[i].regmap[r]&63)==rt2[i+1]) wont_dirty_i|=1<<r;
7372               if(branch_regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
7373             }
7374           }
7375           if(wr) {
7376             #ifndef DESTRUCTIVE_WRITEBACK
7377             branch_regs[i].dirty&=wont_dirty_i;
7378             #endif
7379             branch_regs[i].dirty|=will_dirty_i;
7380           }
7381         }
7382       }
7383     }
7384     else if(itype[i]==SYSCALL)
7385     {
7386       // SYSCALL instruction (software interrupt)
7387       will_dirty_i=0;
7388       wont_dirty_i=0;
7389     }
7390     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
7391     {
7392       // ERET instruction (return from interrupt)
7393       will_dirty_i=0;
7394       wont_dirty_i=0;
7395     }
7396     will_dirty_next=will_dirty_i;
7397     wont_dirty_next=wont_dirty_i;
7398     for(r=0;r<HOST_REGS;r++) {
7399       if(r!=EXCLUDE_REG) {
7400         if((regs[i].regmap[r]&63)==rt1[i]) will_dirty_i|=1<<r;
7401         if((regs[i].regmap[r]&63)==rt2[i]) will_dirty_i|=1<<r;
7402         if((regs[i].regmap[r]&63)>33) will_dirty_i&=~(1<<r);
7403         if(regs[i].regmap[r]<=0) will_dirty_i&=~(1<<r);
7404         if(regs[i].regmap[r]==CCREG) will_dirty_i|=1<<r;
7405         if((regs[i].regmap[r]&63)==rt1[i]) wont_dirty_i|=1<<r;
7406         if((regs[i].regmap[r]&63)==rt2[i]) wont_dirty_i|=1<<r;
7407         if(regs[i].regmap[r]==CCREG) wont_dirty_i|=1<<r;
7408         if(i>istart) {
7409           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=FJUMP)
7410           {
7411             // Don't store a register immediately after writing it,
7412             // may prevent dual-issue.
7413             if((regs[i].regmap[r]&63)==rt1[i-1]) wont_dirty_i|=1<<r;
7414             if((regs[i].regmap[r]&63)==rt2[i-1]) wont_dirty_i|=1<<r;
7415           }
7416         }
7417       }
7418     }
7419     // Save it
7420     will_dirty[i]=will_dirty_i;
7421     wont_dirty[i]=wont_dirty_i;
7422     // Mark registers that won't be dirtied as not dirty
7423     if(wr) {
7424       /*DebugMessage(M64MSG_VERBOSE, "wr (%d,%d) %x will:",istart,iend,start+i*4);
7425       for(r=0;r<HOST_REGS;r++) {
7426         if((will_dirty_i>>r)&1) {
7427           DebugMessage(M64MSG_VERBOSE, " r%d",r);
7428         }
7429       }*/
7430
7431       //if(i==istart||(itype[i-1]!=RJUMP&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=FJUMP)) {
7432         regs[i].dirty|=will_dirty_i;
7433         #ifndef DESTRUCTIVE_WRITEBACK
7434         regs[i].dirty&=wont_dirty_i;
7435         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
7436         {
7437           if(i<iend-1&&itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
7438             for(r=0;r<HOST_REGS;r++) {
7439               if(r!=EXCLUDE_REG) {
7440                 if(regs[i].regmap[r]==regmap_pre[i+2][r]) {
7441                   regs[i+2].wasdirty&=wont_dirty_i|~(1<<r);
7442                 }else {/*DebugMessage(M64MSG_VERBOSE, "i: %x (%d) mismatch(+2): %d",start+i*4,i,r); / *assert(!((wont_dirty_i>>r)&1));*/}
7443               }
7444             }
7445           }
7446         }
7447         else
7448         {
7449           if(i<iend) {
7450             for(r=0;r<HOST_REGS;r++) {
7451               if(r!=EXCLUDE_REG) {
7452                 if(regs[i].regmap[r]==regmap_pre[i+1][r]) {
7453                   regs[i+1].wasdirty&=wont_dirty_i|~(1<<r);
7454                 }else {/*DebugMessage(M64MSG_VERBOSE, "i: %x (%d) mismatch(+1): %d",start+i*4,i,r);/ *assert(!((wont_dirty_i>>r)&1));*/}
7455               }
7456             }
7457           }
7458         }
7459         #endif
7460       //}
7461     }
7462     // Deal with changed mappings
7463     temp_will_dirty=will_dirty_i;
7464     temp_wont_dirty=wont_dirty_i;
7465     for(r=0;r<HOST_REGS;r++) {
7466       if(r!=EXCLUDE_REG) {
7467         int nr;
7468         if(regs[i].regmap[r]==regmap_pre[i][r]) {
7469           if(wr) {
7470             #ifndef DESTRUCTIVE_WRITEBACK
7471             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
7472             #endif
7473             regs[i].wasdirty|=will_dirty_i&(1<<r);
7474           }
7475         }
7476         else if((nr=get_reg(regs[i].regmap,regmap_pre[i][r]))>=0) {
7477           // Register moved to a different register
7478           will_dirty_i&=~(1<<r);
7479           wont_dirty_i&=~(1<<r);
7480           will_dirty_i|=((temp_will_dirty>>nr)&1)<<r;
7481           wont_dirty_i|=((temp_wont_dirty>>nr)&1)<<r;
7482           if(wr) {
7483             #ifndef DESTRUCTIVE_WRITEBACK
7484             regs[i].wasdirty&=wont_dirty_i|~(1<<r);
7485             #endif
7486             regs[i].wasdirty|=will_dirty_i&(1<<r);
7487           }
7488         }
7489         else {
7490           will_dirty_i&=~(1<<r);
7491           wont_dirty_i&=~(1<<r);
7492           if((regmap_pre[i][r]&63)>0 && (regmap_pre[i][r]&63)<34) {
7493             will_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
7494             wont_dirty_i|=((unneeded_reg[i]>>(regmap_pre[i][r]&63))&1)<<r;
7495           } else {
7496             wont_dirty_i|=1<<r;
7497             /*DebugMessage(M64MSG_VERBOSE, "i: %x (%d) mismatch: %d",start+i*4,i,r);/ *assert(!((will_dirty>>r)&1));*/
7498           }
7499         }
7500       }
7501     }
7502   }
7503 }
7504
7505 #ifdef ASSEM_DEBUG
7506   /* disassembly */
7507 static void disassemble_inst(int i)
7508 {
7509     if (bt[i]) DebugMessage(M64MSG_VERBOSE, "*"); else DebugMessage(M64MSG_VERBOSE, " ");
7510     switch(itype[i]) {
7511       case UJUMP:
7512         printf (" %x: %s %8x",start+i*4,insn[i],ba[i]);break;
7513       case CJUMP:
7514         printf (" %x: %s r%d,r%d,%8x",start+i*4,insn[i],rs1[i],rs2[i],i?start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14):*ba);break;
7515       case SJUMP:
7516         printf (" %x: %s r%d,%8x",start+i*4,insn[i],rs1[i],start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14));break;
7517       case FJUMP:
7518         printf (" %x: %s %8x",start+i*4,insn[i],ba[i]);break;
7519       case RJUMP:
7520         if ((opcode2[i]&1)&&rt1[i]!=31)
7521           printf (" %x: %s r%d,r%d",start+i*4,insn[i],rt1[i],rs1[i]);
7522         else
7523           printf (" %x: %s r%d",start+i*4,insn[i],rs1[i]);
7524         break;
7525       case SPAN:
7526         printf (" %x: %s (pagespan) r%d,r%d,%8x",start+i*4,insn[i],rs1[i],rs2[i],ba[i]);break;
7527       case IMM16:
7528         if(opcode[i]==0xf) //LUI
7529           printf (" %x: %s r%d,%4x0000",start+i*4,insn[i],rt1[i],imm[i]&0xffff);
7530         else
7531           printf (" %x: %s r%d,r%d,%d",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
7532         break;
7533       case LOAD:
7534       case LOADLR:
7535         printf (" %x: %s r%d,r%d+%x",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
7536         break;
7537       case STORE:
7538       case STORELR:
7539         printf (" %x: %s r%d,r%d+%x",start+i*4,insn[i],rs2[i],rs1[i],imm[i]);
7540         break;
7541       case ALU:
7542       case SHIFT:
7543         printf (" %x: %s r%d,r%d,r%d",start+i*4,insn[i],rt1[i],rs1[i],rs2[i]);
7544         break;
7545       case MULTDIV:
7546         printf (" %x: %s r%d,r%d",start+i*4,insn[i],rs1[i],rs2[i]);
7547         break;
7548       case SHIFTIMM:
7549         printf (" %x: %s r%d,r%d,%d",start+i*4,insn[i],rt1[i],rs1[i],imm[i]);
7550         break;
7551       case MOV:
7552         if((opcode2[i]&0x1d)==0x10)
7553           printf (" %x: %s r%d",start+i*4,insn[i],rt1[i]);
7554         else if((opcode2[i]&0x1d)==0x11)
7555           printf (" %x: %s r%d",start+i*4,insn[i],rs1[i]);
7556         else
7557           printf (" %x: %s",start+i*4,insn[i]);
7558         break;
7559       case COP0:
7560         if(opcode2[i]==0)
7561           printf (" %x: %s r%d,cpr0[%d]",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC0
7562         else if(opcode2[i]==4)
7563           printf (" %x: %s r%d,cpr0[%d]",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC0
7564         else printf (" %x: %s",start+i*4,insn[i]);
7565         break;
7566       case COP1:
7567         if(opcode2[i]<3)
7568           printf (" %x: %s r%d,cpr1[%d]",start+i*4,insn[i],rt1[i],(source[i]>>11)&0x1f); // MFC1
7569         else if(opcode2[i]>3)
7570           printf (" %x: %s r%d,cpr1[%d]",start+i*4,insn[i],rs1[i],(source[i]>>11)&0x1f); // MTC1
7571         else printf (" %x: %s",start+i*4,insn[i]);
7572         break;
7573       case C1LS:
7574         printf (" %x: %s cpr1[%d],r%d+%x",start+i*4,insn[i],(source[i]>>16)&0x1f,rs1[i],imm[i]);
7575         break;
7576       default:
7577         //printf (" %s %8x",insn[i],source[i]);
7578         printf (" %x: %s",start+i*4,insn[i]);
7579     }
7580 }
7581 #endif
7582
7583 void new_dynarec_init()
7584 {
7585   DebugMessage(M64MSG_INFO, "Init new dynarec");
7586
7587 #if NEW_DYNAREC == NEW_DYNAREC_ARM
7588   if ((base_addr = mmap ((u_char *)BASE_ADDR, 1<<TARGET_SIZE_2,
7589             PROT_READ | PROT_WRITE | PROT_EXEC,
7590             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
7591             -1, 0)) <= 0) {DebugMessage(M64MSG_ERROR, "mmap() failed");}
7592 #else
7593   if ((base_addr = mmap (NULL, 1<<TARGET_SIZE_2,
7594             PROT_READ | PROT_WRITE | PROT_EXEC,
7595             MAP_PRIVATE | MAP_ANONYMOUS,
7596             -1, 0)) <= 0) {DebugMessage(M64MSG_ERROR, "mmap() failed");}
7597 #endif
7598   out=(u_char *)base_addr;
7599
7600   rdword=&readmem_dword;
7601   fake_pc.f.r.rs=(long long int *)&readmem_dword;
7602   fake_pc.f.r.rt=(long long int *)&readmem_dword;
7603   fake_pc.f.r.rd=(long long int *)&readmem_dword;
7604   int n;
7605   for(n=0x80000;n<0x80800;n++)
7606     invalid_code[n]=1;
7607   for(n=0;n<65536;n++)
7608     hash_table[n][0]=hash_table[n][2]=-1;
7609   memset(mini_ht,-1,sizeof(mini_ht));
7610   memset(restore_candidate,0,sizeof(restore_candidate));
7611   copy=shadow;
7612   expirep=16384; // Expiry pointer, +2 blocks
7613   pending_exception=0;
7614   literalcount=0;
7615 #ifdef HOST_IMM8
7616   // Copy this into local area so we don't have to put it in every literal pool
7617   invc_ptr=invalid_code;
7618 #endif
7619   stop_after_jal=0;
7620   // TLB
7621   using_tlb=0;
7622   for(n=0;n<524288;n++) // 0 .. 0x7FFFFFFF
7623     memory_map[n]=-1;
7624   for(n=524288;n<526336;n++) // 0x80000000 .. 0x807FFFFF
7625     memory_map[n]=((u_int)rdram-0x80000000)>>2;
7626   for(n=526336;n<1048576;n++) // 0x80800000 .. 0xFFFFFFFF
7627     memory_map[n]=-1;
7628   for(n=0;n<0x8000;n++) { // 0 .. 0x7FFFFFFF
7629     writemem[n] = write_nomem_new;
7630     writememb[n] = write_nomemb_new;
7631     writememh[n] = write_nomemh_new;
7632     writememd[n] = write_nomemd_new;
7633     readmem[n] = read_nomem_new;
7634     readmemb[n] = read_nomemb_new;
7635     readmemh[n] = read_nomemh_new;
7636     readmemd[n] = read_nomemd_new;
7637   }
7638   for(n=0x8000;n<0x8080;n++) { // 0x80000000 .. 0x807FFFFF
7639     writemem[n] = write_rdram_new;
7640     writememb[n] = write_rdramb_new;
7641     writememh[n] = write_rdramh_new;
7642     writememd[n] = write_rdramd_new;
7643   }
7644   for(n=0xC000;n<0x10000;n++) { // 0xC0000000 .. 0xFFFFFFFF
7645     writemem[n] = write_nomem_new;
7646     writememb[n] = write_nomemb_new;
7647     writememh[n] = write_nomemh_new;
7648     writememd[n] = write_nomemd_new;
7649     readmem[n] = read_nomem_new;
7650     readmemb[n] = read_nomemb_new;
7651     readmemh[n] = read_nomemh_new;
7652     readmemd[n] = read_nomemd_new;
7653   }
7654   tlb_hacks();
7655   arch_init();
7656 }
7657
7658 void new_dynarec_cleanup()
7659 {
7660   int n;
7661   if (munmap (base_addr, 1<<TARGET_SIZE_2) < 0) {DebugMessage(M64MSG_ERROR, "munmap() failed");}
7662   for(n=0;n<4096;n++) ll_clear(jump_in+n);
7663   for(n=0;n<4096;n++) ll_clear(jump_out+n);
7664   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
7665   #ifdef ROM_COPY
7666   if (munmap (ROM_COPY, 67108864) < 0) {DebugMessage(M64MSG_ERROR, "munmap() failed");}
7667   #endif
7668 }
7669
7670 int new_recompile_block(int addr)
7671 {
7672 /*
7673   if(addr==0x800cd050) {
7674     int block;
7675     for(block=0x80000;block<0x80800;block++) invalidate_block(block);
7676     int n;
7677     for(n=0;n<=2048;n++) ll_clear(jump_dirty+n);
7678   }
7679 */
7680   //if(Count==365117028) tracedebug=1;
7681   assem_debug("NOTCOMPILED: addr = %x -> %x", (int)addr, (int)out);
7682 #if defined (COUNT_NOTCOMPILEDS )
7683   notcompiledCount++;
7684   log_message( "notcompiledCount=%i", notcompiledCount );
7685 #endif
7686   //DebugMessage(M64MSG_VERBOSE, "NOTCOMPILED: addr = %x -> %x", (int)addr, (int)out);
7687   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (compile %x)",Count,next_interupt,addr);
7688   //if(debug)
7689   //DebugMessage(M64MSG_VERBOSE, "TRACE: count=%d next=%d (checksum %x)",Count,next_interupt,mchecksum());
7690   //DebugMessage(M64MSG_VERBOSE, "fpu mapping=%x enabled=%x",(Status & 0x04000000)>>26,(Status & 0x20000000)>>29);
7691   /*if(Count>=312978186) {
7692     rlist();
7693   }*/
7694   //rlist();
7695   start = (u_int)addr&~3;
7696   //assert(((u_int)addr&1)==0);
7697   if ((int)addr >= 0xa4000000 && (int)addr < 0xa4001000) {
7698     source = (u_int *)((u_int)SP_DMEM+start-0xa4000000);
7699     pagelimit = 0xa4001000;
7700   }
7701   else if ((int)addr >= 0x80000000 && (int)addr < 0x80800000) {
7702     source = (u_int *)((u_int)rdram+start-0x80000000);
7703     pagelimit = 0x80800000;
7704   }
7705   else if ((signed int)addr >= (signed int)0xC0000000) {
7706     //DebugMessage(M64MSG_VERBOSE, "addr=%x mm=%x",(u_int)addr,(memory_map[start>>12]<<2));
7707     //if(tlb_LUT_r[start>>12])
7708       //source = (u_int *)(((int)rdram)+(tlb_LUT_r[start>>12]&0xFFFFF000)+(((int)addr)&0xFFF)-0x80000000);
7709     if((signed int)memory_map[start>>12]>=0) {
7710       source = (u_int *)((u_int)(start+(memory_map[start>>12]<<2)));
7711       pagelimit=(start+4096)&0xFFFFF000;
7712       int map=memory_map[start>>12];
7713       int i;
7714       for(i=0;i<5;i++) {
7715         //DebugMessage(M64MSG_VERBOSE, "start: %x next: %x",map,memory_map[pagelimit>>12]);
7716         if((map&0xBFFFFFFF)==(memory_map[pagelimit>>12]&0xBFFFFFFF)) pagelimit+=4096;
7717       }
7718       assem_debug("pagelimit=%x",pagelimit);
7719       assem_debug("mapping=%x (%x)",memory_map[start>>12],(memory_map[start>>12]<<2)+start);
7720     }
7721     else {
7722       assem_debug("Compile at unmapped memory address: %x ", (int)addr);
7723       //assem_debug("start: %x next: %x",memory_map[start>>12],memory_map[(start+4096)>>12]);
7724       return 1; // Caller will invoke exception handler
7725     }
7726     //DebugMessage(M64MSG_VERBOSE, "source= %x",(int)source);
7727   }
7728   else {
7729     //DebugMessage(M64MSG_VERBOSE, "Compile at bogus memory address: %x ", (int)addr);
7730     log_message("Compile at bogus memory address: %x", (int)addr);
7731     exit(1);
7732   }
7733
7734   /* Pass 1: disassemble */
7735   /* Pass 2: register dependencies, branch targets */
7736   /* Pass 3: register allocation */
7737   /* Pass 4: branch dependencies */
7738   /* Pass 5: pre-alloc */
7739   /* Pass 6: optimize clean/dirty state */
7740   /* Pass 7: flag 32-bit registers */
7741   /* Pass 8: assembly */
7742   /* Pass 9: linker */
7743   /* Pass 10: garbage collection / free memory */
7744
7745   int i,j;
7746   int done=0;
7747   unsigned int type,op,op2;
7748
7749   //DebugMessage(M64MSG_VERBOSE, "addr = %x source = %x %x", addr,source,source[0]);
7750
7751   /* Pass 1 disassembly */
7752
7753   for(i=0;!done;i++) {
7754     bt[i]=0;likely[i]=0;ooo[i]=0;op2=0;
7755     minimum_free_regs[i]=0;
7756     opcode[i]=op=source[i]>>26;
7757     switch(op)
7758     {
7759       case 0x00: strcpy(insn[i],"special"); type=NI;
7760         op2=source[i]&0x3f;
7761         switch(op2)
7762         {
7763           case 0x00: strcpy(insn[i],"SLL"); type=SHIFTIMM; break;
7764           case 0x02: strcpy(insn[i],"SRL"); type=SHIFTIMM; break;
7765           case 0x03: strcpy(insn[i],"SRA"); type=SHIFTIMM; break;
7766           case 0x04: strcpy(insn[i],"SLLV"); type=SHIFT; break;
7767           case 0x06: strcpy(insn[i],"SRLV"); type=SHIFT; break;
7768           case 0x07: strcpy(insn[i],"SRAV"); type=SHIFT; break;
7769           case 0x08: strcpy(insn[i],"JR"); type=RJUMP; break;
7770           case 0x09: strcpy(insn[i],"JALR"); type=RJUMP; break;
7771           case 0x0C: strcpy(insn[i],"SYSCALL"); type=SYSCALL; break;
7772           case 0x0D: strcpy(insn[i],"BREAK"); type=OTHER; break;
7773           case 0x0F: strcpy(insn[i],"SYNC"); type=OTHER; break;
7774           case 0x10: strcpy(insn[i],"MFHI"); type=MOV; break;
7775           case 0x11: strcpy(insn[i],"MTHI"); type=MOV; break;
7776           case 0x12: strcpy(insn[i],"MFLO"); type=MOV; break;
7777           case 0x13: strcpy(insn[i],"MTLO"); type=MOV; break;
7778           case 0x14: strcpy(insn[i],"DSLLV"); type=SHIFT; break;
7779           case 0x16: strcpy(insn[i],"DSRLV"); type=SHIFT; break;
7780           case 0x17: strcpy(insn[i],"DSRAV"); type=SHIFT; break;
7781           case 0x18: strcpy(insn[i],"MULT"); type=MULTDIV; break;
7782           case 0x19: strcpy(insn[i],"MULTU"); type=MULTDIV; break;
7783           case 0x1A: strcpy(insn[i],"DIV"); type=MULTDIV; break;
7784           case 0x1B: strcpy(insn[i],"DIVU"); type=MULTDIV; break;
7785           case 0x1C: strcpy(insn[i],"DMULT"); type=MULTDIV; break;
7786           case 0x1D: strcpy(insn[i],"DMULTU"); type=MULTDIV; break;
7787           case 0x1E: strcpy(insn[i],"DDIV"); type=MULTDIV; break;
7788           case 0x1F: strcpy(insn[i],"DDIVU"); type=MULTDIV; break;
7789           case 0x20: strcpy(insn[i],"ADD"); type=ALU; break;
7790           case 0x21: strcpy(insn[i],"ADDU"); type=ALU; break;
7791           case 0x22: strcpy(insn[i],"SUB"); type=ALU; break;
7792           case 0x23: strcpy(insn[i],"SUBU"); type=ALU; break;
7793           case 0x24: strcpy(insn[i],"AND"); type=ALU; break;
7794           case 0x25: strcpy(insn[i],"OR"); type=ALU; break;
7795           case 0x26: strcpy(insn[i],"XOR"); type=ALU; break;
7796           case 0x27: strcpy(insn[i],"NOR"); type=ALU; break;
7797           case 0x2A: strcpy(insn[i],"SLT"); type=ALU; break;
7798           case 0x2B: strcpy(insn[i],"SLTU"); type=ALU; break;
7799           case 0x2C: strcpy(insn[i],"DADD"); type=ALU; break;
7800           case 0x2D: strcpy(insn[i],"DADDU"); type=ALU; break;
7801           case 0x2E: strcpy(insn[i],"DSUB"); type=ALU; break;
7802           case 0x2F: strcpy(insn[i],"DSUBU"); type=ALU; break;
7803           case 0x30: strcpy(insn[i],"TGE"); type=NI; break;
7804           case 0x31: strcpy(insn[i],"TGEU"); type=NI; break;
7805           case 0x32: strcpy(insn[i],"TLT"); type=NI; break;
7806           case 0x33: strcpy(insn[i],"TLTU"); type=NI; break;
7807           case 0x34: strcpy(insn[i],"TEQ"); type=NI; break;
7808           case 0x36: strcpy(insn[i],"TNE"); type=NI; break;
7809           case 0x38: strcpy(insn[i],"DSLL"); type=SHIFTIMM; break;
7810           case 0x3A: strcpy(insn[i],"DSRL"); type=SHIFTIMM; break;
7811           case 0x3B: strcpy(insn[i],"DSRA"); type=SHIFTIMM; break;
7812           case 0x3C: strcpy(insn[i],"DSLL32"); type=SHIFTIMM; break;
7813           case 0x3E: strcpy(insn[i],"DSRL32"); type=SHIFTIMM; break;
7814           case 0x3F: strcpy(insn[i],"DSRA32"); type=SHIFTIMM; break;
7815         }
7816         break;
7817       case 0x01: strcpy(insn[i],"regimm"); type=NI;
7818         op2=(source[i]>>16)&0x1f;
7819         switch(op2)
7820         {
7821           case 0x00: strcpy(insn[i],"BLTZ"); type=SJUMP; break;
7822           case 0x01: strcpy(insn[i],"BGEZ"); type=SJUMP; break;
7823           case 0x02: strcpy(insn[i],"BLTZL"); type=SJUMP; break;
7824           case 0x03: strcpy(insn[i],"BGEZL"); type=SJUMP; break;
7825           case 0x08: strcpy(insn[i],"TGEI"); type=NI; break;
7826           case 0x09: strcpy(insn[i],"TGEIU"); type=NI; break;
7827           case 0x0A: strcpy(insn[i],"TLTI"); type=NI; break;
7828           case 0x0B: strcpy(insn[i],"TLTIU"); type=NI; break;
7829           case 0x0C: strcpy(insn[i],"TEQI"); type=NI; break;
7830           case 0x0E: strcpy(insn[i],"TNEI"); type=NI; break;
7831           case 0x10: strcpy(insn[i],"BLTZAL"); type=SJUMP; break;
7832           case 0x11: strcpy(insn[i],"BGEZAL"); type=SJUMP; break;
7833           case 0x12: strcpy(insn[i],"BLTZALL"); type=SJUMP; break;
7834           case 0x13: strcpy(insn[i],"BGEZALL"); type=SJUMP; break;
7835         }
7836         break;
7837       case 0x02: strcpy(insn[i],"J"); type=UJUMP; break;
7838       case 0x03: strcpy(insn[i],"JAL"); type=UJUMP; break;
7839       case 0x04: strcpy(insn[i],"BEQ"); type=CJUMP; break;
7840       case 0x05: strcpy(insn[i],"BNE"); type=CJUMP; break;
7841       case 0x06: strcpy(insn[i],"BLEZ"); type=CJUMP; break;
7842       case 0x07: strcpy(insn[i],"BGTZ"); type=CJUMP; break;
7843       case 0x08: strcpy(insn[i],"ADDI"); type=IMM16; break;
7844       case 0x09: strcpy(insn[i],"ADDIU"); type=IMM16; break;
7845       case 0x0A: strcpy(insn[i],"SLTI"); type=IMM16; break;
7846       case 0x0B: strcpy(insn[i],"SLTIU"); type=IMM16; break;
7847       case 0x0C: strcpy(insn[i],"ANDI"); type=IMM16; break;
7848       case 0x0D: strcpy(insn[i],"ORI"); type=IMM16; break;
7849       case 0x0E: strcpy(insn[i],"XORI"); type=IMM16; break;
7850       case 0x0F: strcpy(insn[i],"LUI"); type=IMM16; break;
7851       case 0x10: strcpy(insn[i],"cop0"); type=NI;
7852         op2=(source[i]>>21)&0x1f;
7853         switch(op2)
7854         {
7855           case 0x00: strcpy(insn[i],"MFC0"); type=COP0; break;
7856           case 0x04: strcpy(insn[i],"MTC0"); type=COP0; break;
7857           case 0x10: strcpy(insn[i],"tlb"); type=NI;
7858           switch(source[i]&0x3f)
7859           {
7860             case 0x01: strcpy(insn[i],"TLBR"); type=COP0; break;
7861             case 0x02: strcpy(insn[i],"TLBWI"); type=COP0; break;
7862             case 0x06: strcpy(insn[i],"TLBWR"); type=COP0; break;
7863             case 0x08: strcpy(insn[i],"TLBP"); type=COP0; break;
7864             case 0x18: strcpy(insn[i],"ERET"); type=COP0; break;
7865           }
7866         }
7867         break;
7868       case 0x11: strcpy(insn[i],"cop1"); type=NI;
7869         op2=(source[i]>>21)&0x1f;
7870         switch(op2)
7871         {
7872           case 0x00: strcpy(insn[i],"MFC1"); type=COP1; break;
7873           case 0x01: strcpy(insn[i],"DMFC1"); type=COP1; break;
7874           case 0x02: strcpy(insn[i],"CFC1"); type=COP1; break;
7875           case 0x04: strcpy(insn[i],"MTC1"); type=COP1; break;
7876           case 0x05: strcpy(insn[i],"DMTC1"); type=COP1; break;
7877           case 0x06: strcpy(insn[i],"CTC1"); type=COP1; break;
7878           case 0x08: strcpy(insn[i],"BC1"); type=FJUMP;
7879           switch((source[i]>>16)&0x3)
7880           {
7881             case 0x00: strcpy(insn[i],"BC1F"); break;
7882             case 0x01: strcpy(insn[i],"BC1T"); break;
7883             case 0x02: strcpy(insn[i],"BC1FL"); break;
7884             case 0x03: strcpy(insn[i],"BC1TL"); break;
7885           }
7886           break;
7887           case 0x10: strcpy(insn[i],"C1.S"); type=NI;
7888           switch(source[i]&0x3f)
7889           {
7890             case 0x00: strcpy(insn[i],"ADD.S"); type=FLOAT; break;
7891             case 0x01: strcpy(insn[i],"SUB.S"); type=FLOAT; break;
7892             case 0x02: strcpy(insn[i],"MUL.S"); type=FLOAT; break;
7893             case 0x03: strcpy(insn[i],"DIV.S"); type=FLOAT; break;
7894             case 0x04: strcpy(insn[i],"SQRT.S"); type=FLOAT; break;
7895             case 0x05: strcpy(insn[i],"ABS.S"); type=FLOAT; break;
7896             case 0x06: strcpy(insn[i],"MOV.S"); type=FLOAT; break;
7897             case 0x07: strcpy(insn[i],"NEG.S"); type=FLOAT; break;
7898             case 0x08: strcpy(insn[i],"ROUND.L.S"); type=FCONV; break;
7899             case 0x09: strcpy(insn[i],"TRUNC.L.S"); type=FCONV; break;
7900             case 0x0A: strcpy(insn[i],"CEIL.L.S"); type=FCONV; break;
7901             case 0x0B: strcpy(insn[i],"FLOOR.L.S"); type=FCONV; break;
7902             case 0x0C: strcpy(insn[i],"ROUND.W.S"); type=FCONV; break;
7903             case 0x0D: strcpy(insn[i],"TRUNC.W.S"); type=FCONV; break;
7904             case 0x0E: strcpy(insn[i],"CEIL.W.S"); type=FCONV; break;
7905             case 0x0F: strcpy(insn[i],"FLOOR.W.S"); type=FCONV; break;
7906             case 0x21: strcpy(insn[i],"CVT.D.S"); type=FCONV; break;
7907             case 0x24: strcpy(insn[i],"CVT.W.S"); type=FCONV; break;
7908             case 0x25: strcpy(insn[i],"CVT.L.S"); type=FCONV; break;
7909             case 0x30: strcpy(insn[i],"C.F.S"); type=FCOMP; break;
7910             case 0x31: strcpy(insn[i],"C.UN.S"); type=FCOMP; break;
7911             case 0x32: strcpy(insn[i],"C.EQ.S"); type=FCOMP; break;
7912             case 0x33: strcpy(insn[i],"C.UEQ.S"); type=FCOMP; break;
7913             case 0x34: strcpy(insn[i],"C.OLT.S"); type=FCOMP; break;
7914             case 0x35: strcpy(insn[i],"C.ULT.S"); type=FCOMP; break;
7915             case 0x36: strcpy(insn[i],"C.OLE.S"); type=FCOMP; break;
7916             case 0x37: strcpy(insn[i],"C.ULE.S"); type=FCOMP; break;
7917             case 0x38: strcpy(insn[i],"C.SF.S"); type=FCOMP; break;
7918             case 0x39: strcpy(insn[i],"C.NGLE.S"); type=FCOMP; break;
7919             case 0x3A: strcpy(insn[i],"C.SEQ.S"); type=FCOMP; break;
7920             case 0x3B: strcpy(insn[i],"C.NGL.S"); type=FCOMP; break;
7921             case 0x3C: strcpy(insn[i],"C.LT.S"); type=FCOMP; break;
7922             case 0x3D: strcpy(insn[i],"C.NGE.S"); type=FCOMP; break;
7923             case 0x3E: strcpy(insn[i],"C.LE.S"); type=FCOMP; break;
7924             case 0x3F: strcpy(insn[i],"C.NGT.S"); type=FCOMP; break;
7925           }
7926           break;
7927           case 0x11: strcpy(insn[i],"C1.D"); type=NI;
7928           switch(source[i]&0x3f)
7929           {
7930             case 0x00: strcpy(insn[i],"ADD.D"); type=FLOAT; break;
7931             case 0x01: strcpy(insn[i],"SUB.D"); type=FLOAT; break;
7932             case 0x02: strcpy(insn[i],"MUL.D"); type=FLOAT; break;
7933             case 0x03: strcpy(insn[i],"DIV.D"); type=FLOAT; break;
7934             case 0x04: strcpy(insn[i],"SQRT.D"); type=FLOAT; break;
7935             case 0x05: strcpy(insn[i],"ABS.D"); type=FLOAT; break;
7936             case 0x06: strcpy(insn[i],"MOV.D"); type=FLOAT; break;
7937             case 0x07: strcpy(insn[i],"NEG.D"); type=FLOAT; break;
7938             case 0x08: strcpy(insn[i],"ROUND.L.D"); type=FCONV; break;
7939             case 0x09: strcpy(insn[i],"TRUNC.L.D"); type=FCONV; break;
7940             case 0x0A: strcpy(insn[i],"CEIL.L.D"); type=FCONV; break;
7941             case 0x0B: strcpy(insn[i],"FLOOR.L.D"); type=FCONV; break;
7942             case 0x0C: strcpy(insn[i],"ROUND.W.D"); type=FCONV; break;
7943             case 0x0D: strcpy(insn[i],"TRUNC.W.D"); type=FCONV; break;
7944             case 0x0E: strcpy(insn[i],"CEIL.W.D"); type=FCONV; break;
7945             case 0x0F: strcpy(insn[i],"FLOOR.W.D"); type=FCONV; break;
7946             case 0x20: strcpy(insn[i],"CVT.S.D"); type=FCONV; break;
7947             case 0x24: strcpy(insn[i],"CVT.W.D"); type=FCONV; break;
7948             case 0x25: strcpy(insn[i],"CVT.L.D"); type=FCONV; break;
7949             case 0x30: strcpy(insn[i],"C.F.D"); type=FCOMP; break;
7950             case 0x31: strcpy(insn[i],"C.UN.D"); type=FCOMP; break;
7951             case 0x32: strcpy(insn[i],"C.EQ.D"); type=FCOMP; break;
7952             case 0x33: strcpy(insn[i],"C.UEQ.D"); type=FCOMP; break;
7953             case 0x34: strcpy(insn[i],"C.OLT.D"); type=FCOMP; break;
7954             case 0x35: strcpy(insn[i],"C.ULT.D"); type=FCOMP; break;
7955             case 0x36: strcpy(insn[i],"C.OLE.D"); type=FCOMP; break;
7956             case 0x37: strcpy(insn[i],"C.ULE.D"); type=FCOMP; break;
7957             case 0x38: strcpy(insn[i],"C.SF.D"); type=FCOMP; break;
7958             case 0x39: strcpy(insn[i],"C.NGLE.D"); type=FCOMP; break;
7959             case 0x3A: strcpy(insn[i],"C.SEQ.D"); type=FCOMP; break;
7960             case 0x3B: strcpy(insn[i],"C.NGL.D"); type=FCOMP; break;
7961             case 0x3C: strcpy(insn[i],"C.LT.D"); type=FCOMP; break;
7962             case 0x3D: strcpy(insn[i],"C.NGE.D"); type=FCOMP; break;
7963             case 0x3E: strcpy(insn[i],"C.LE.D"); type=FCOMP; break;
7964             case 0x3F: strcpy(insn[i],"C.NGT.D"); type=FCOMP; break;
7965           }
7966           break;
7967           case 0x14: strcpy(insn[i],"C1.W"); type=NI;
7968           switch(source[i]&0x3f)
7969           {
7970             case 0x20: strcpy(insn[i],"CVT.S.W"); type=FCONV; break;
7971             case 0x21: strcpy(insn[i],"CVT.D.W"); type=FCONV; break;
7972           }
7973           break;
7974           case 0x15: strcpy(insn[i],"C1.L"); type=NI;
7975           switch(source[i]&0x3f)
7976           {
7977             case 0x20: strcpy(insn[i],"CVT.S.L"); type=FCONV; break;
7978             case 0x21: strcpy(insn[i],"CVT.D.L"); type=FCONV; break;
7979           }
7980           break;
7981         }
7982         break;
7983       case 0x14: strcpy(insn[i],"BEQL"); type=CJUMP; break;
7984       case 0x15: strcpy(insn[i],"BNEL"); type=CJUMP; break;
7985       case 0x16: strcpy(insn[i],"BLEZL"); type=CJUMP; break;
7986       case 0x17: strcpy(insn[i],"BGTZL"); type=CJUMP; break;
7987       case 0x18: strcpy(insn[i],"DADDI"); type=IMM16; break;
7988       case 0x19: strcpy(insn[i],"DADDIU"); type=IMM16; break;
7989       case 0x1A: strcpy(insn[i],"LDL"); type=LOADLR; break;
7990       case 0x1B: strcpy(insn[i],"LDR"); type=LOADLR; break;
7991       case 0x20: strcpy(insn[i],"LB"); type=LOAD; break;
7992       case 0x21: strcpy(insn[i],"LH"); type=LOAD; break;
7993       case 0x22: strcpy(insn[i],"LWL"); type=LOADLR; break;
7994       case 0x23: strcpy(insn[i],"LW"); type=LOAD; break;
7995       case 0x24: strcpy(insn[i],"LBU"); type=LOAD; break;
7996       case 0x25: strcpy(insn[i],"LHU"); type=LOAD; break;
7997       case 0x26: strcpy(insn[i],"LWR"); type=LOADLR; break;
7998       case 0x27: strcpy(insn[i],"LWU"); type=LOAD; break;
7999       case 0x28: strcpy(insn[i],"SB"); type=STORE; break;
8000       case 0x29: strcpy(insn[i],"SH"); type=STORE; break;
8001       case 0x2A: strcpy(insn[i],"SWL"); type=STORELR; break;
8002       case 0x2B: strcpy(insn[i],"SW"); type=STORE; break;
8003       case 0x2C: strcpy(insn[i],"SDL"); type=STORELR; break;
8004       case 0x2D: strcpy(insn[i],"SDR"); type=STORELR; break;
8005       case 0x2E: strcpy(insn[i],"SWR"); type=STORELR; break;
8006       case 0x2F: strcpy(insn[i],"CACHE"); type=NOP; break;
8007       case 0x30: strcpy(insn[i],"LL"); type=NI; break;
8008       case 0x31: strcpy(insn[i],"LWC1"); type=C1LS; break;
8009       case 0x34: strcpy(insn[i],"LLD"); type=NI; break;
8010       case 0x35: strcpy(insn[i],"LDC1"); type=C1LS; break;
8011       case 0x37: strcpy(insn[i],"LD"); type=LOAD; break;
8012       case 0x38: strcpy(insn[i],"SC"); type=NI; break;
8013       case 0x39: strcpy(insn[i],"SWC1"); type=C1LS; break;
8014       case 0x3C: strcpy(insn[i],"SCD"); type=NI; break;
8015       case 0x3D: strcpy(insn[i],"SDC1"); type=C1LS; break;
8016       case 0x3F: strcpy(insn[i],"SD"); type=STORE; break;
8017       default: strcpy(insn[i],"???"); type=NI; break;
8018     }
8019     itype[i]=type;
8020     opcode2[i]=op2;
8021     /* Get registers/immediates */
8022     lt1[i]=0;
8023     us1[i]=0;
8024     us2[i]=0;
8025     dep1[i]=0;
8026     dep2[i]=0;
8027     switch(type) {
8028       case LOAD:
8029         rs1[i]=(source[i]>>21)&0x1f;
8030         rs2[i]=0;
8031         rt1[i]=(source[i]>>16)&0x1f;
8032         rt2[i]=0;
8033         imm[i]=(short)source[i];
8034         break;
8035       case STORE:
8036       case STORELR:
8037         rs1[i]=(source[i]>>21)&0x1f;
8038         rs2[i]=(source[i]>>16)&0x1f;
8039         rt1[i]=0;
8040         rt2[i]=0;
8041         imm[i]=(short)source[i];
8042         if(op==0x2c||op==0x2d||op==0x3f) us1[i]=rs2[i]; // 64-bit SDL/SDR/SD
8043         break;
8044       case LOADLR:
8045         // LWL/LWR only load part of the register,
8046         // therefore the target register must be treated as a source too
8047         rs1[i]=(source[i]>>21)&0x1f;
8048         rs2[i]=(source[i]>>16)&0x1f;
8049         rt1[i]=(source[i]>>16)&0x1f;
8050         rt2[i]=0;
8051         imm[i]=(short)source[i];
8052         if(op==0x1a||op==0x1b) us1[i]=rs2[i]; // LDR/LDL
8053         if(op==0x26) dep1[i]=rt1[i]; // LWR
8054         break;
8055       case IMM16:
8056         if (op==0x0f) rs1[i]=0; // LUI instruction has no source register
8057         else rs1[i]=(source[i]>>21)&0x1f;
8058         rs2[i]=0;
8059         rt1[i]=(source[i]>>16)&0x1f;
8060         rt2[i]=0;
8061         if(op>=0x0c&&op<=0x0e) { // ANDI/ORI/XORI
8062           imm[i]=(unsigned short)source[i];
8063         }else{
8064           imm[i]=(short)source[i];
8065         }
8066         if(op==0x18||op==0x19) us1[i]=rs1[i]; // DADDI/DADDIU
8067         if(op==0x0a||op==0x0b) us1[i]=rs1[i]; // SLTI/SLTIU
8068         if(op==0x0d||op==0x0e) dep1[i]=rs1[i]; // ORI/XORI
8069         break;
8070       case UJUMP:
8071         rs1[i]=0;
8072         rs2[i]=0;
8073         rt1[i]=0;
8074         rt2[i]=0;
8075         // The JAL instruction writes to r31.
8076         if (op&1) {
8077           rt1[i]=31;
8078         }
8079         rs2[i]=CCREG;
8080         break;
8081       case RJUMP:
8082         rs1[i]=(source[i]>>21)&0x1f;
8083         rs2[i]=0;
8084         rt1[i]=0;
8085         rt2[i]=0;
8086         // The JALR instruction writes to rd.
8087         if (op2&1) {
8088           rt1[i]=(source[i]>>11)&0x1f;
8089         }
8090         rs2[i]=CCREG;
8091         break;
8092       case CJUMP:
8093         rs1[i]=(source[i]>>21)&0x1f;
8094         rs2[i]=(source[i]>>16)&0x1f;
8095         rt1[i]=0;
8096         rt2[i]=0;
8097         if(op&2) { // BGTZ/BLEZ
8098           rs2[i]=0;
8099         }
8100         us1[i]=rs1[i];
8101         us2[i]=rs2[i];
8102         likely[i]=op>>4;
8103         break;
8104       case SJUMP:
8105         rs1[i]=(source[i]>>21)&0x1f;
8106         rs2[i]=CCREG;
8107         rt1[i]=0;
8108         rt2[i]=0;
8109         us1[i]=rs1[i];
8110         if(op2&0x10) { // BxxAL
8111           rt1[i]=31;
8112           // NOTE: If the branch is not taken, r31 is still overwritten
8113         }
8114         likely[i]=(op2&2)>>1;
8115         break;
8116       case FJUMP:
8117         rs1[i]=FSREG;
8118         rs2[i]=CSREG;
8119         rt1[i]=0;
8120         rt2[i]=0;
8121         likely[i]=((source[i])>>17)&1;
8122         break;
8123       case ALU:
8124         rs1[i]=(source[i]>>21)&0x1f; // source
8125         rs2[i]=(source[i]>>16)&0x1f; // subtract amount
8126         rt1[i]=(source[i]>>11)&0x1f; // destination
8127         rt2[i]=0;
8128         if(op2==0x2a||op2==0x2b) { // SLT/SLTU
8129           us1[i]=rs1[i];us2[i]=rs2[i];
8130         }
8131         else if(op2>=0x24&&op2<=0x27) { // AND/OR/XOR/NOR
8132           dep1[i]=rs1[i];dep2[i]=rs2[i];
8133         }
8134         else if(op2>=0x2c&&op2<=0x2f) { // DADD/DSUB
8135           dep1[i]=rs1[i];dep2[i]=rs2[i];
8136         }
8137         break;
8138       case MULTDIV:
8139         rs1[i]=(source[i]>>21)&0x1f; // source
8140         rs2[i]=(source[i]>>16)&0x1f; // divisor
8141         rt1[i]=HIREG;
8142         rt2[i]=LOREG;
8143         if (op2>=0x1c&&op2<=0x1f) { // DMULT/DMULTU/DDIV/DDIVU
8144           us1[i]=rs1[i];us2[i]=rs2[i];
8145         }
8146         break;
8147       case MOV:
8148         rs1[i]=0;
8149         rs2[i]=0;
8150         rt1[i]=0;
8151         rt2[i]=0;
8152         if(op2==0x10) rs1[i]=HIREG; // MFHI
8153         if(op2==0x11) rt1[i]=HIREG; // MTHI
8154         if(op2==0x12) rs1[i]=LOREG; // MFLO
8155         if(op2==0x13) rt1[i]=LOREG; // MTLO
8156         if((op2&0x1d)==0x10) rt1[i]=(source[i]>>11)&0x1f; // MFxx
8157         if((op2&0x1d)==0x11) rs1[i]=(source[i]>>21)&0x1f; // MTxx
8158         dep1[i]=rs1[i];
8159         break;
8160       case SHIFT:
8161         rs1[i]=(source[i]>>16)&0x1f; // target of shift
8162         rs2[i]=(source[i]>>21)&0x1f; // shift amount
8163         rt1[i]=(source[i]>>11)&0x1f; // destination
8164         rt2[i]=0;
8165         // DSLLV/DSRLV/DSRAV are 64-bit
8166         if(op2>=0x14&&op2<=0x17) us1[i]=rs1[i];
8167         break;
8168       case SHIFTIMM:
8169         rs1[i]=(source[i]>>16)&0x1f;
8170         rs2[i]=0;
8171         rt1[i]=(source[i]>>11)&0x1f;
8172         rt2[i]=0;
8173         imm[i]=(source[i]>>6)&0x1f;
8174         // DSxx32 instructions
8175         if(op2>=0x3c) imm[i]|=0x20;
8176         // DSLL/DSRL/DSRA/DSRA32/DSRL32 but not DSLL32 require 64-bit source
8177         if(op2>=0x38&&op2!=0x3c) us1[i]=rs1[i];
8178         break;
8179       case COP0:
8180         rs1[i]=0;
8181         rs2[i]=0;
8182         rt1[i]=0;
8183         rt2[i]=0;
8184         if(op2==0) rt1[i]=(source[i]>>16)&0x1F; // MFC0
8185         if(op2==4) rs1[i]=(source[i]>>16)&0x1F; // MTC0
8186         if(op2==4&&((source[i]>>11)&0x1f)==12) rt2[i]=CSREG; // Status
8187         if(op2==16) if((source[i]&0x3f)==0x18) rs2[i]=CCREG; // ERET
8188         break;
8189       case COP1:
8190         rs1[i]=0;
8191         rs2[i]=0;
8192         rt1[i]=0;
8193         rt2[i]=0;
8194         if(op2<3) rt1[i]=(source[i]>>16)&0x1F; // MFC1/DMFC1/CFC1
8195         if(op2>3) rs1[i]=(source[i]>>16)&0x1F; // MTC1/DMTC1/CTC1
8196         if(op2==5) us1[i]=rs1[i]; // DMTC1
8197         rs2[i]=CSREG;
8198         break;
8199       case C1LS:
8200         rs1[i]=(source[i]>>21)&0x1F;
8201         rs2[i]=CSREG;
8202         rt1[i]=0;
8203         rt2[i]=0;
8204         imm[i]=(short)source[i];
8205         break;
8206       case FLOAT:
8207       case FCONV:
8208         rs1[i]=0;
8209         rs2[i]=CSREG;
8210         rt1[i]=0;
8211         rt2[i]=0;
8212         break;
8213       case FCOMP:
8214         rs1[i]=FSREG;
8215         rs2[i]=CSREG;
8216         rt1[i]=FSREG;
8217         rt2[i]=0;
8218         break;
8219       case SYSCALL:
8220         rs1[i]=CCREG;
8221         rs2[i]=0;
8222         rt1[i]=0;
8223         rt2[i]=0;
8224         break;
8225       default:
8226         rs1[i]=0;
8227         rs2[i]=0;
8228         rt1[i]=0;
8229         rt2[i]=0;
8230     }
8231     /* Calculate branch target addresses */
8232     if(type==UJUMP)
8233       ba[i]=((start+i*4+4)&0xF0000000)|(((unsigned int)source[i]<<6)>>4);
8234     else if(type==CJUMP&&rs1[i]==rs2[i]&&(op&1))
8235       ba[i]=start+i*4+8; // Ignore never taken branch
8236     else if(type==SJUMP&&rs1[i]==0&&!(op2&1))
8237       ba[i]=start+i*4+8; // Ignore never taken branch
8238     else if(type==CJUMP||type==SJUMP||type==FJUMP)
8239       ba[i]=start+i*4+4+((signed int)((unsigned int)source[i]<<16)>>14);
8240     else ba[i]=-1;
8241     /* Is this the end of the block? */
8242     if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)) {
8243       if(rt1[i-1]==0) { // Continue past subroutine call (JAL)
8244         done=1;
8245         // Does the block continue due to a branch?
8246         for(j=i-1;j>=0;j--)
8247         {
8248           if(ba[j]==start+i*4) done=j=0; // Branch into delay slot
8249           if(ba[j]==start+i*4+4) done=j=0;
8250           if(ba[j]==start+i*4+8) done=j=0;
8251         }
8252       }
8253       else {
8254         if(stop_after_jal) done=1;
8255         // Stop on BREAK
8256         if((source[i+1]&0xfc00003f)==0x0d) done=1;
8257       }
8258       // Don't recompile stuff that's already compiled
8259       if(check_addr(start+i*4+4)) done=1;
8260       // Don't get too close to the limit
8261       if(i>MAXBLOCK/2) done=1;
8262     }
8263     if(i>0&&itype[i-1]==SYSCALL&&stop_after_jal) done=1;
8264     assert(i<MAXBLOCK-1);
8265     if(start+i*4==pagelimit-4) done=1;
8266     assert(start+i*4<pagelimit);
8267     if (i==MAXBLOCK-1) done=1;
8268     // Stop if we're compiling junk
8269     if(itype[i]==NI&&opcode[i]==0x11) {
8270       done=stop_after_jal=1;
8271       DebugMessage(M64MSG_VERBOSE, "Disabled speculative precompilation");
8272     }
8273   }
8274   slen=i;
8275   if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==RJUMP||itype[i-1]==FJUMP) {
8276     if(start+i*4==pagelimit) {
8277       itype[i-1]=SPAN;
8278     }
8279   }
8280   assert(slen>0);
8281
8282   /* Pass 2 - Register dependencies and branch targets */
8283
8284   unneeded_registers(0,slen-1,0);
8285
8286   /* Pass 3 - Register allocation */
8287
8288   struct regstat current; // Current register allocations/status
8289   current.is32=1;
8290   current.dirty=0;
8291   current.u=unneeded_reg[0];
8292   current.uu=unneeded_reg_upper[0];
8293   clear_all_regs(current.regmap);
8294   alloc_reg(&current,0,CCREG);
8295   dirty_reg(&current,CCREG);
8296   current.isconst=0;
8297   current.wasconst=0;
8298   int ds=0;
8299   int cc=0;
8300   int hr;
8301
8302   provisional_32bit();
8303
8304   if((u_int)addr&1) {
8305     // First instruction is delay slot
8306     cc=-1;
8307     bt[1]=1;
8308     ds=1;
8309     unneeded_reg[0]=1;
8310     unneeded_reg_upper[0]=1;
8311     current.regmap[HOST_BTREG]=BTREG;
8312   }
8313
8314   for(i=0;i<slen;i++)
8315   {
8316     if(bt[i])
8317     {
8318       int hr;
8319       for(hr=0;hr<HOST_REGS;hr++)
8320       {
8321         // Is this really necessary?
8322         if(current.regmap[hr]==0) current.regmap[hr]=-1;
8323       }
8324       current.isconst=0;
8325     }
8326     if(i>1)
8327     {
8328       if((opcode[i-2]&0x2f)==0x05) // BNE/BNEL
8329       {
8330         if(rs1[i-2]==0||rs2[i-2]==0)
8331         {
8332           if(rs1[i-2]) {
8333             current.is32|=1LL<<rs1[i-2];
8334             int hr=get_reg(current.regmap,rs1[i-2]|64);
8335             if(hr>=0) current.regmap[hr]=-1;
8336           }
8337           if(rs2[i-2]) {
8338             current.is32|=1LL<<rs2[i-2];
8339             int hr=get_reg(current.regmap,rs2[i-2]|64);
8340             if(hr>=0) current.regmap[hr]=-1;
8341           }
8342         }
8343       }
8344     }
8345     // If something jumps here with 64-bit values
8346     // then promote those registers to 64 bits
8347     if(bt[i])
8348     {
8349       uint64_t temp_is32=current.is32;
8350       for(j=i-1;j>=0;j--)
8351       {
8352         if(ba[j]==start+i*4)
8353           temp_is32&=branch_regs[j].is32;
8354       }
8355       for(j=i;j<slen;j++)
8356       {
8357         if(ba[j]==start+i*4)
8358           //temp_is32=1;
8359           temp_is32&=p32[j];
8360       }
8361       if(temp_is32!=current.is32) {
8362         //DebugMessage(M64MSG_VERBOSE, "dumping 32-bit regs (%x)",start+i*4);
8363         #ifndef DESTRUCTIVE_WRITEBACK
8364         if(ds)
8365         #endif
8366         for(hr=0;hr<HOST_REGS;hr++)
8367         {
8368           int r=current.regmap[hr];
8369           if(r>0&&r<64)
8370           {
8371             if((current.dirty>>hr)&((current.is32&~temp_is32)>>r)&1) {
8372               temp_is32|=1LL<<r;
8373               //DebugMessage(M64MSG_VERBOSE, "restore %d",r);
8374             }
8375           }
8376         }
8377         current.is32=temp_is32;
8378       }
8379     }
8380     memcpy(regmap_pre[i],current.regmap,sizeof(current.regmap));
8381     regs[i].wasconst=current.isconst;
8382     regs[i].was32=current.is32;
8383     regs[i].wasdirty=current.dirty;
8384     #ifdef DESTRUCTIVE_WRITEBACK
8385     // To change a dirty register from 32 to 64 bits, we must write
8386     // it out during the previous cycle (for branches, 2 cycles)
8387     if(i<slen-1&&bt[i+1]&&itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP&&itype[i-1]!=FJUMP)
8388     {
8389       uint64_t temp_is32=current.is32;
8390       for(j=i-1;j>=0;j--)
8391       {
8392         if(ba[j]==start+i*4+4)
8393           temp_is32&=branch_regs[j].is32;
8394       }
8395       for(j=i;j<slen;j++)
8396       {
8397         if(ba[j]==start+i*4+4)
8398           //temp_is32=1;
8399           temp_is32&=p32[j];
8400       }
8401       if(temp_is32!=current.is32) {
8402         //DebugMessage(M64MSG_VERBOSE, "pre-dumping 32-bit regs (%x)",start+i*4);
8403         for(hr=0;hr<HOST_REGS;hr++)
8404         {
8405           int r=current.regmap[hr];
8406           if(r>0)
8407           {
8408             if((current.dirty>>hr)&((current.is32&~temp_is32)>>(r&63))&1) {
8409               if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP&&itype[i]!=FJUMP)
8410               {
8411                 if(rs1[i]!=(r&63)&&rs2[i]!=(r&63))
8412                 {
8413                   //DebugMessage(M64MSG_VERBOSE, "dump %d/r%d",hr,r);
8414                   current.regmap[hr]=-1;
8415                   if(get_reg(current.regmap,r|64)>=0)
8416                     current.regmap[get_reg(current.regmap,r|64)]=-1;
8417                 }
8418               }
8419             }
8420           }
8421         }
8422       }
8423     }
8424     else if(i<slen-2&&bt[i+2]&&(source[i-1]>>16)!=0x1000&&(itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP))
8425     {
8426       uint64_t temp_is32=current.is32;
8427       for(j=i-1;j>=0;j--)
8428       {
8429         if(ba[j]==start+i*4+8)
8430           temp_is32&=branch_regs[j].is32;
8431       }
8432       for(j=i;j<slen;j++)
8433       {
8434         if(ba[j]==start+i*4+8)
8435           //temp_is32=1;
8436           temp_is32&=p32[j];
8437       }
8438       if(temp_is32!=current.is32) {
8439         //DebugMessage(M64MSG_VERBOSE, "pre-dumping 32-bit regs (%x)",start+i*4);
8440         for(hr=0;hr<HOST_REGS;hr++)
8441         {
8442           int r=current.regmap[hr];
8443           if(r>0)
8444           {
8445             if((current.dirty>>hr)&((current.is32&~temp_is32)>>(r&63))&1) {
8446               if(rs1[i]!=(r&63)&&rs2[i]!=(r&63)&&rs1[i+1]!=(r&63)&&rs2[i+1]!=(r&63))
8447               {
8448                 //DebugMessage(M64MSG_VERBOSE, "dump %d/r%d",hr,r);
8449                 current.regmap[hr]=-1;
8450                 if(get_reg(current.regmap,r|64)>=0)
8451                   current.regmap[get_reg(current.regmap,r|64)]=-1;
8452               }
8453             }
8454           }
8455         }
8456       }
8457     }
8458     #endif
8459     if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP&&itype[i]!=FJUMP) {
8460       if(i+1<slen) {
8461         current.u=unneeded_reg[i+1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
8462         current.uu=unneeded_reg_upper[i+1]&~((1LL<<us1[i])|(1LL<<us2[i]));
8463         if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
8464         current.u|=1;
8465         current.uu|=1;
8466       } else {
8467         current.u=1;
8468         current.uu=1;
8469       }
8470     } else {
8471       if(i+1<slen) {
8472         current.u=branch_unneeded_reg[i]&~((1LL<<rs1[i+1])|(1LL<<rs2[i+1]));
8473         current.uu=branch_unneeded_reg_upper[i]&~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
8474         if((~current.uu>>rt1[i+1])&1) current.uu&=~((1LL<<dep1[i+1])|(1LL<<dep2[i+1]));
8475         current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
8476         current.uu&=~((1LL<<us1[i])|(1LL<<us2[i]));
8477         current.u|=1;
8478         current.uu|=1;
8479       } else { DebugMessage(M64MSG_ERROR, "oops, branch at end of block with no delay slot");exit(1); }
8480     }
8481     is_ds[i]=ds;
8482     if(ds) {
8483       ds=0; // Skip delay slot, already allocated as part of branch
8484       // ...but we need to alloc it in case something jumps here
8485       if(i+1<slen) {
8486         current.u=branch_unneeded_reg[i-1]&unneeded_reg[i+1];
8487         current.uu=branch_unneeded_reg_upper[i-1]&unneeded_reg_upper[i+1];
8488       }else{
8489         current.u=branch_unneeded_reg[i-1];
8490         current.uu=branch_unneeded_reg_upper[i-1];
8491       }
8492       current.u&=~((1LL<<rs1[i])|(1LL<<rs2[i]));
8493       current.uu&=~((1LL<<us1[i])|(1LL<<us2[i]));
8494       if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
8495       current.u|=1;
8496       current.uu|=1;
8497       struct regstat temp;
8498       memcpy(&temp,&current,sizeof(current));
8499       temp.wasdirty=temp.dirty;
8500       temp.was32=temp.is32;
8501       // TODO: Take into account unconditional branches, as below
8502       delayslot_alloc(&temp,i);
8503       memcpy(regs[i].regmap,temp.regmap,sizeof(temp.regmap));
8504       regs[i].wasdirty=temp.wasdirty;
8505       regs[i].was32=temp.was32;
8506       regs[i].dirty=temp.dirty;
8507       regs[i].is32=temp.is32;
8508       regs[i].isconst=0;
8509       regs[i].wasconst=0;
8510       current.isconst=0;
8511       // Create entry (branch target) regmap
8512       for(hr=0;hr<HOST_REGS;hr++)
8513       {
8514         int r=temp.regmap[hr];
8515         if(r>=0) {
8516           if(r!=regmap_pre[i][hr]) {
8517             regs[i].regmap_entry[hr]=-1;
8518           }
8519           else
8520           {
8521             if(r<64){
8522               if((current.u>>r)&1) {
8523                 regs[i].regmap_entry[hr]=-1;
8524                 regs[i].regmap[hr]=-1;
8525                 //Don't clear regs in the delay slot as the branch might need them
8526                 //current.regmap[hr]=-1;
8527               }else
8528                 regs[i].regmap_entry[hr]=r;
8529             }
8530             else {
8531               if((current.uu>>(r&63))&1) {
8532                 regs[i].regmap_entry[hr]=-1;
8533                 regs[i].regmap[hr]=-1;
8534                 //Don't clear regs in the delay slot as the branch might need them
8535                 //current.regmap[hr]=-1;
8536               }else
8537                 regs[i].regmap_entry[hr]=r;
8538             }
8539           }
8540         } else {
8541           // First instruction expects CCREG to be allocated
8542           if(i==0&&hr==HOST_CCREG)
8543             regs[i].regmap_entry[hr]=CCREG;
8544           else
8545             regs[i].regmap_entry[hr]=-1;
8546         }
8547       }
8548     }
8549     else { // Not delay slot
8550       switch(itype[i]) {
8551         case UJUMP:
8552           //current.isconst=0; // DEBUG
8553           //current.wasconst=0; // DEBUG
8554           //regs[i].wasconst=0; // DEBUG
8555           clear_const(&current,rt1[i]);
8556           alloc_cc(&current,i);
8557           dirty_reg(&current,CCREG);
8558           if (rt1[i]==31) {
8559             alloc_reg(&current,i,31);
8560             dirty_reg(&current,31);
8561             assert(rs1[i+1]!=31&&rs2[i+1]!=31);
8562             #ifdef REG_PREFETCH
8563             alloc_reg(&current,i,PTEMP);
8564             #endif
8565             //current.is32|=1LL<<rt1[i];
8566           }
8567           ooo[i]=1;
8568           delayslot_alloc(&current,i+1);
8569           //current.isconst=0; // DEBUG
8570           ds=1;
8571           //DebugMessage(M64MSG_VERBOSE, "i=%d, isconst=%x",i,current.isconst);
8572           break;
8573         case RJUMP:
8574           //current.isconst=0;
8575           //current.wasconst=0;
8576           //regs[i].wasconst=0;
8577           clear_const(&current,rs1[i]);
8578           clear_const(&current,rt1[i]);
8579           alloc_cc(&current,i);
8580           dirty_reg(&current,CCREG);
8581           if(rs1[i]!=rt1[i+1]&&rs1[i]!=rt2[i+1]) {
8582             alloc_reg(&current,i,rs1[i]);
8583             if (rt1[i]!=0) {
8584               alloc_reg(&current,i,rt1[i]);
8585               dirty_reg(&current,rt1[i]);
8586               assert(rs1[i+1]!=31&&rs2[i+1]!=31);
8587               #ifdef REG_PREFETCH
8588               alloc_reg(&current,i,PTEMP);
8589               #endif
8590             }
8591             #ifdef USE_MINI_HT
8592             if(rs1[i]==31) { // JALR
8593               alloc_reg(&current,i,RHASH);
8594               #ifndef HOST_IMM_ADDR32
8595               alloc_reg(&current,i,RHTBL);
8596               #endif
8597             }
8598             #endif
8599             delayslot_alloc(&current,i+1);
8600           } else {
8601             // The delay slot overwrites our source register,
8602             // allocate a temporary register to hold the old value.
8603             current.isconst=0;
8604             current.wasconst=0;
8605             regs[i].wasconst=0;
8606             delayslot_alloc(&current,i+1);
8607             current.isconst=0;
8608             alloc_reg(&current,i,RTEMP);
8609           }
8610           //current.isconst=0; // DEBUG
8611           ooo[i]=1;
8612           ds=1;
8613           break;
8614         case CJUMP:
8615           //current.isconst=0;
8616           //current.wasconst=0;
8617           //regs[i].wasconst=0;
8618           clear_const(&current,rs1[i]);
8619           clear_const(&current,rs2[i]);
8620           if((opcode[i]&0x3E)==4) // BEQ/BNE
8621           {
8622             alloc_cc(&current,i);
8623             dirty_reg(&current,CCREG);
8624             if(rs1[i]) alloc_reg(&current,i,rs1[i]);
8625             if(rs2[i]) alloc_reg(&current,i,rs2[i]);
8626             if(!((current.is32>>rs1[i])&(current.is32>>rs2[i])&1))
8627             {
8628               if(rs1[i]) alloc_reg64(&current,i,rs1[i]);
8629               if(rs2[i]) alloc_reg64(&current,i,rs2[i]);
8630             }
8631             if((rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1]))||
8632                (rs2[i]&&(rs2[i]==rt1[i+1]||rs2[i]==rt2[i+1]))) {
8633               // The delay slot overwrites one of our conditions.
8634               // Allocate the branch condition registers instead.
8635               current.isconst=0;
8636               current.wasconst=0;
8637               regs[i].wasconst=0;
8638               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
8639               if(rs2[i]) alloc_reg(&current,i,rs2[i]);
8640               if(!((current.is32>>rs1[i])&(current.is32>>rs2[i])&1))
8641               {
8642                 if(rs1[i]) alloc_reg64(&current,i,rs1[i]);
8643                 if(rs2[i]) alloc_reg64(&current,i,rs2[i]);
8644               }
8645             }
8646             else
8647             {
8648               ooo[i]=1;
8649               delayslot_alloc(&current,i+1);
8650             }
8651           }
8652           else
8653           if((opcode[i]&0x3E)==6) // BLEZ/BGTZ
8654           {
8655             alloc_cc(&current,i);
8656             dirty_reg(&current,CCREG);
8657             alloc_reg(&current,i,rs1[i]);
8658             if(!(current.is32>>rs1[i]&1))
8659             {
8660               alloc_reg64(&current,i,rs1[i]);
8661             }
8662             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
8663               // The delay slot overwrites one of our conditions.
8664               // Allocate the branch condition registers instead.
8665               current.isconst=0;
8666               current.wasconst=0;
8667               regs[i].wasconst=0;
8668               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
8669               if(!((current.is32>>rs1[i])&1))
8670               {
8671                 if(rs1[i]) alloc_reg64(&current,i,rs1[i]);
8672               }
8673             }
8674             else
8675             {
8676               ooo[i]=1;
8677               delayslot_alloc(&current,i+1);
8678             }
8679           }
8680           else
8681           // Don't alloc the delay slot yet because we might not execute it
8682           if((opcode[i]&0x3E)==0x14) // BEQL/BNEL
8683           {
8684             current.isconst=0;
8685             current.wasconst=0;
8686             regs[i].wasconst=0;
8687             alloc_cc(&current,i);
8688             dirty_reg(&current,CCREG);
8689             alloc_reg(&current,i,rs1[i]);
8690             alloc_reg(&current,i,rs2[i]);
8691             if(!((current.is32>>rs1[i])&(current.is32>>rs2[i])&1))
8692             {
8693               alloc_reg64(&current,i,rs1[i]);
8694               alloc_reg64(&current,i,rs2[i]);
8695             }
8696           }
8697           else
8698           if((opcode[i]&0x3E)==0x16) // BLEZL/BGTZL
8699           {
8700             current.isconst=0;
8701             current.wasconst=0;
8702             regs[i].wasconst=0;
8703             alloc_cc(&current,i);
8704             dirty_reg(&current,CCREG);
8705             alloc_reg(&current,i,rs1[i]);
8706             if(!(current.is32>>rs1[i]&1))
8707             {
8708               alloc_reg64(&current,i,rs1[i]);
8709             }
8710           }
8711           ds=1;
8712           //current.isconst=0;
8713           break;
8714         case SJUMP:
8715           //current.isconst=0;
8716           //current.wasconst=0;
8717           //regs[i].wasconst=0;
8718           clear_const(&current,rs1[i]);
8719           clear_const(&current,rt1[i]);
8720           //if((opcode2[i]&0x1E)==0x0) // BLTZ/BGEZ
8721           if((opcode2[i]&0x0E)==0x0) // BLTZ/BGEZ
8722           {
8723             alloc_cc(&current,i);
8724             dirty_reg(&current,CCREG);
8725             alloc_reg(&current,i,rs1[i]);
8726             if(!(current.is32>>rs1[i]&1))
8727             {
8728               alloc_reg64(&current,i,rs1[i]);
8729             }
8730             if (rt1[i]==31) { // BLTZAL/BGEZAL
8731               alloc_reg(&current,i,31);
8732               dirty_reg(&current,31);
8733               assert(rs1[i+1]!=31&&rs2[i+1]!=31);
8734               //#ifdef REG_PREFETCH
8735               //alloc_reg(&current,i,PTEMP);
8736               //#endif
8737               //current.is32|=1LL<<rt1[i];
8738             }
8739             if(rs1[i]&&(rs1[i]==rt1[i+1]||rs1[i]==rt2[i+1])) {
8740               // The delay slot overwrites the branch condition.
8741               // Allocate the branch condition registers instead.
8742               current.isconst=0;
8743               current.wasconst=0;
8744               regs[i].wasconst=0;
8745               if(rs1[i]) alloc_reg(&current,i,rs1[i]);
8746               if(!((current.is32>>rs1[i])&1))
8747               {
8748                 if(rs1[i]) alloc_reg64(&current,i,rs1[i]);
8749               }
8750             }
8751             else
8752             {
8753               ooo[i]=1;
8754               delayslot_alloc(&current,i+1);
8755             }
8756           }
8757           else
8758           // Don't alloc the delay slot yet because we might not execute it
8759           if((opcode2[i]&0x1E)==0x2) // BLTZL/BGEZL
8760           {
8761             current.isconst=0;
8762             current.wasconst=0;
8763             regs[i].wasconst=0;
8764             alloc_cc(&current,i);
8765             dirty_reg(&current,CCREG);
8766             alloc_reg(&current,i,rs1[i]);
8767             if(!(current.is32>>rs1[i]&1))
8768             {
8769               alloc_reg64(&current,i,rs1[i]);
8770             }
8771           }
8772           ds=1;
8773           //current.isconst=0;
8774           break;
8775         case FJUMP:
8776           current.isconst=0;
8777           current.wasconst=0;
8778           regs[i].wasconst=0;
8779           if(likely[i]==0) // BC1F/BC1T
8780           {
8781             // TODO: Theoretically we can run out of registers here on x86.
8782             // The delay slot can allocate up to six, and we need to check
8783             // CSREG before executing the delay slot.  Possibly we can drop
8784             // the cycle count and then reload it after checking that the
8785             // FPU is in a usable state, or don't do out-of-order execution.
8786             alloc_cc(&current,i);
8787             dirty_reg(&current,CCREG);
8788             alloc_reg(&current,i,FSREG);
8789             alloc_reg(&current,i,CSREG);
8790             if(itype[i+1]==FCOMP) {
8791               // The delay slot overwrites the branch condition.
8792               // Allocate the branch condition registers instead.
8793               alloc_cc(&current,i);
8794               dirty_reg(&current,CCREG);
8795               alloc_reg(&current,i,CSREG);
8796               alloc_reg(&current,i,FSREG);
8797             }
8798             else {
8799               ooo[i]=1;
8800               delayslot_alloc(&current,i+1);
8801               alloc_reg(&current,i+1,CSREG);
8802             }
8803           }
8804           else
8805           // Don't alloc the delay slot yet because we might not execute it
8806           if(likely[i]) // BC1FL/BC1TL
8807           {
8808             alloc_cc(&current,i);
8809             dirty_reg(&current,CCREG);
8810             alloc_reg(&current,i,CSREG);
8811             alloc_reg(&current,i,FSREG);
8812           }
8813           ds=1;
8814           current.isconst=0;
8815           break;
8816         case IMM16:
8817           imm16_alloc(&current,i);
8818           break;
8819         case LOAD:
8820         case LOADLR:
8821           load_alloc(&current,i);
8822           break;
8823         case STORE:
8824         case STORELR:
8825           store_alloc(&current,i);
8826           break;
8827         case ALU:
8828           alu_alloc(&current,i);
8829           break;
8830         case SHIFT:
8831           shift_alloc(&current,i);
8832           break;
8833         case MULTDIV:
8834           multdiv_alloc(&current,i);
8835           break;
8836         case SHIFTIMM:
8837           shiftimm_alloc(&current,i);
8838           break;
8839         case MOV:
8840           mov_alloc(&current,i);
8841           break;
8842         case COP0:
8843           cop0_alloc(&current,i);
8844           break;
8845         case COP1:
8846           cop1_alloc(&current,i);
8847           break;
8848         case C1LS:
8849           c1ls_alloc(&current,i);
8850           break;
8851         case FCONV:
8852           fconv_alloc(&current,i);
8853           break;
8854         case FLOAT:
8855           float_alloc(&current,i);
8856           break;
8857         case FCOMP:
8858           fcomp_alloc(&current,i);
8859           break;
8860         case SYSCALL:
8861           syscall_alloc(&current,i);
8862           break;
8863         case SPAN:
8864           pagespan_alloc(&current,i);
8865           break;
8866       }
8867
8868       // Drop the upper half of registers that have become 32-bit
8869       current.uu|=current.is32&((1LL<<rt1[i])|(1LL<<rt2[i]));
8870       if(itype[i]!=UJUMP&&itype[i]!=CJUMP&&itype[i]!=SJUMP&&itype[i]!=RJUMP&&itype[i]!=FJUMP) {
8871         current.uu&=~((1LL<<us1[i])|(1LL<<us2[i]));
8872         if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
8873         current.uu|=1;
8874       } else {
8875         current.uu|=current.is32&((1LL<<rt1[i+1])|(1LL<<rt2[i+1]));
8876         current.uu&=~((1LL<<us1[i+1])|(1LL<<us2[i+1]));
8877         if((~current.uu>>rt1[i+1])&1) current.uu&=~((1LL<<dep1[i+1])|(1LL<<dep2[i+1]));
8878         current.uu&=~((1LL<<us1[i])|(1LL<<us2[i]));
8879         current.uu|=1;
8880       }
8881
8882       // Create entry (branch target) regmap
8883       for(hr=0;hr<HOST_REGS;hr++)
8884       {
8885         int r,or;
8886         r=current.regmap[hr];
8887         if(r>=0) {
8888           if(r!=regmap_pre[i][hr]) {
8889             // TODO: delay slot (?)
8890             or=get_reg(regmap_pre[i],r); // Get old mapping for this register
8891             if(or<0||(r&63)>=TEMPREG){
8892               regs[i].regmap_entry[hr]=-1;
8893             }
8894             else
8895             {
8896               // Just move it to a different register
8897               regs[i].regmap_entry[hr]=r;
8898               // If it was dirty before, it's still dirty
8899               if((regs[i].wasdirty>>or)&1) dirty_reg(&current,r&63);
8900             }
8901           }
8902           else
8903           {
8904             // Unneeded
8905             if(r==0){
8906               regs[i].regmap_entry[hr]=0;
8907             }
8908             else
8909             if(r<64){
8910               if((current.u>>r)&1) {
8911                 regs[i].regmap_entry[hr]=-1;
8912                 //regs[i].regmap[hr]=-1;
8913                 current.regmap[hr]=-1;
8914               }else
8915                 regs[i].regmap_entry[hr]=r;
8916             }
8917             else {
8918               if((current.uu>>(r&63))&1) {
8919                 regs[i].regmap_entry[hr]=-1;
8920                 //regs[i].regmap[hr]=-1;
8921                 current.regmap[hr]=-1;
8922               }else
8923                 regs[i].regmap_entry[hr]=r;
8924             }
8925           }
8926         } else {
8927           // Branches expect CCREG to be allocated at the target
8928           if(regmap_pre[i][hr]==CCREG)
8929             regs[i].regmap_entry[hr]=CCREG;
8930           else
8931             regs[i].regmap_entry[hr]=-1;
8932         }
8933       }
8934       memcpy(regs[i].regmap,current.regmap,sizeof(current.regmap));
8935     }
8936     /* Branch post-alloc */
8937     if(i>0)
8938     {
8939       current.was32=current.is32;
8940       current.wasdirty=current.dirty;
8941       switch(itype[i-1]) {
8942         case UJUMP:
8943           memcpy(&branch_regs[i-1],&current,sizeof(current));
8944           branch_regs[i-1].isconst=0;
8945           branch_regs[i-1].wasconst=0;
8946           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
8947           branch_regs[i-1].uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i-1])|(1LL<<us2[i-1]));
8948           alloc_cc(&branch_regs[i-1],i-1);
8949           dirty_reg(&branch_regs[i-1],CCREG);
8950           if(rt1[i-1]==31) { // JAL
8951             alloc_reg(&branch_regs[i-1],i-1,31);
8952             dirty_reg(&branch_regs[i-1],31);
8953             branch_regs[i-1].is32|=1LL<<31;
8954           }
8955           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
8956           memcpy(constmap[i],constmap[i-1],sizeof(current.constmap));
8957           break;
8958         case RJUMP:
8959           memcpy(&branch_regs[i-1],&current,sizeof(current));
8960           branch_regs[i-1].isconst=0;
8961           branch_regs[i-1].wasconst=0;
8962           branch_regs[i-1].u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
8963           branch_regs[i-1].uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i-1])|(1LL<<us2[i-1]));
8964           alloc_cc(&branch_regs[i-1],i-1);
8965           dirty_reg(&branch_regs[i-1],CCREG);
8966           alloc_reg(&branch_regs[i-1],i-1,rs1[i-1]);
8967           if(rt1[i-1]!=0) { // JALR
8968             alloc_reg(&branch_regs[i-1],i-1,rt1[i-1]);
8969             dirty_reg(&branch_regs[i-1],rt1[i-1]);
8970             branch_regs[i-1].is32|=1LL<<rt1[i-1];
8971           }
8972           #ifdef USE_MINI_HT
8973           if(rs1[i-1]==31) { // JALR
8974             alloc_reg(&branch_regs[i-1],i-1,RHASH);
8975             #ifndef HOST_IMM_ADDR32
8976             alloc_reg(&branch_regs[i-1],i-1,RHTBL);
8977             #endif
8978           }
8979           #endif
8980           memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
8981           memcpy(constmap[i],constmap[i-1],sizeof(current.constmap));
8982           break;
8983         case CJUMP:
8984           if((opcode[i-1]&0x3E)==4) // BEQ/BNE
8985           {
8986             alloc_cc(&current,i-1);
8987             dirty_reg(&current,CCREG);
8988             if((rs1[i-1]&&(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]))||
8989                (rs2[i-1]&&(rs2[i-1]==rt1[i]||rs2[i-1]==rt2[i]))) {
8990               // The delay slot overwrote one of our conditions
8991               // Delay slot goes after the test (in order)
8992               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
8993               current.uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i]));
8994               if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
8995               current.u|=1;
8996               current.uu|=1;
8997               delayslot_alloc(&current,i);
8998               current.isconst=0;
8999             }
9000             else
9001             {
9002               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i-1])|(1LL<<rs2[i-1]));
9003               current.uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i-1])|(1LL<<us2[i-1]));
9004               // Alloc the branch condition registers
9005               if(rs1[i-1]) alloc_reg(&current,i-1,rs1[i-1]);
9006               if(rs2[i-1]) alloc_reg(&current,i-1,rs2[i-1]);
9007               if(!((current.is32>>rs1[i-1])&(current.is32>>rs2[i-1])&1))
9008               {
9009                 if(rs1[i-1]) alloc_reg64(&current,i-1,rs1[i-1]);
9010                 if(rs2[i-1]) alloc_reg64(&current,i-1,rs2[i-1]);
9011               }
9012             }
9013             memcpy(&branch_regs[i-1],&current,sizeof(current));
9014             branch_regs[i-1].isconst=0;
9015             branch_regs[i-1].wasconst=0;
9016             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
9017             memcpy(constmap[i],constmap[i-1],sizeof(current.constmap));
9018           }
9019           else
9020           if((opcode[i-1]&0x3E)==6) // BLEZ/BGTZ
9021           {
9022             alloc_cc(&current,i-1);
9023             dirty_reg(&current,CCREG);
9024             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
9025               // The delay slot overwrote the branch condition
9026               // Delay slot goes after the test (in order)
9027               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
9028               current.uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i]));
9029               if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
9030               current.u|=1;
9031               current.uu|=1;
9032               delayslot_alloc(&current,i);
9033               current.isconst=0;
9034             }
9035             else
9036             {
9037               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
9038               current.uu=branch_unneeded_reg_upper[i-1]&~(1LL<<us1[i-1]);
9039               // Alloc the branch condition register
9040               alloc_reg(&current,i-1,rs1[i-1]);
9041               if(!(current.is32>>rs1[i-1]&1))
9042               {
9043                 alloc_reg64(&current,i-1,rs1[i-1]);
9044               }
9045             }
9046             memcpy(&branch_regs[i-1],&current,sizeof(current));
9047             branch_regs[i-1].isconst=0;
9048             branch_regs[i-1].wasconst=0;
9049             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
9050             memcpy(constmap[i],constmap[i-1],sizeof(current.constmap));
9051           }
9052           else
9053           // Alloc the delay slot in case the branch is taken
9054           if((opcode[i-1]&0x3E)==0x14) // BEQL/BNEL
9055           {
9056             memcpy(&branch_regs[i-1],&current,sizeof(current));
9057             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9058             branch_regs[i-1].uu=(branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9059             if((~branch_regs[i-1].uu>>rt1[i])&1) branch_regs[i-1].uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]))|1;
9060             alloc_cc(&branch_regs[i-1],i);
9061             dirty_reg(&branch_regs[i-1],CCREG);
9062             delayslot_alloc(&branch_regs[i-1],i);
9063             branch_regs[i-1].isconst=0;
9064             alloc_reg(&current,i,CCREG); // Not taken path
9065             dirty_reg(&current,CCREG);
9066             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
9067           }
9068           else
9069           if((opcode[i-1]&0x3E)==0x16) // BLEZL/BGTZL
9070           {
9071             memcpy(&branch_regs[i-1],&current,sizeof(current));
9072             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9073             branch_regs[i-1].uu=(branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9074             if((~branch_regs[i-1].uu>>rt1[i])&1) branch_regs[i-1].uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]))|1;
9075             alloc_cc(&branch_regs[i-1],i);
9076             dirty_reg(&branch_regs[i-1],CCREG);
9077             delayslot_alloc(&branch_regs[i-1],i);
9078             branch_regs[i-1].isconst=0;
9079             alloc_reg(&current,i,CCREG); // Not taken path
9080             dirty_reg(&current,CCREG);
9081             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
9082           }
9083           break;
9084         case SJUMP:
9085           //if((opcode2[i-1]&0x1E)==0) // BLTZ/BGEZ
9086           if((opcode2[i-1]&0x0E)==0) // BLTZ/BGEZ
9087           {
9088             alloc_cc(&current,i-1);
9089             dirty_reg(&current,CCREG);
9090             if(rs1[i-1]==rt1[i]||rs1[i-1]==rt2[i]) {
9091               // The delay slot overwrote the branch condition
9092               // Delay slot goes after the test (in order)
9093               current.u=branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i]));
9094               current.uu=branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i]));
9095               if((~current.uu>>rt1[i])&1) current.uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]));
9096               current.u|=1;
9097               current.uu|=1;
9098               delayslot_alloc(&current,i);
9099               current.isconst=0;
9100             }
9101             else
9102             {
9103               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
9104               current.uu=branch_unneeded_reg_upper[i-1]&~(1LL<<us1[i-1]);
9105               // Alloc the branch condition register
9106               alloc_reg(&current,i-1,rs1[i-1]);
9107               if(!(current.is32>>rs1[i-1]&1))
9108               {
9109                 alloc_reg64(&current,i-1,rs1[i-1]);
9110               }
9111             }
9112             memcpy(&branch_regs[i-1],&current,sizeof(current));
9113             branch_regs[i-1].isconst=0;
9114             branch_regs[i-1].wasconst=0;
9115             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
9116             memcpy(constmap[i],constmap[i-1],sizeof(current.constmap));
9117           }
9118           else
9119           // Alloc the delay slot in case the branch is taken
9120           if((opcode2[i-1]&0x1E)==2) // BLTZL/BGEZL
9121           {
9122             memcpy(&branch_regs[i-1],&current,sizeof(current));
9123             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9124             branch_regs[i-1].uu=(branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9125             if((~branch_regs[i-1].uu>>rt1[i])&1) branch_regs[i-1].uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]))|1;
9126             alloc_cc(&branch_regs[i-1],i);
9127             dirty_reg(&branch_regs[i-1],CCREG);
9128             delayslot_alloc(&branch_regs[i-1],i);
9129             branch_regs[i-1].isconst=0;
9130             alloc_reg(&current,i,CCREG); // Not taken path
9131             dirty_reg(&current,CCREG);
9132             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
9133           }
9134           // FIXME: BLTZAL/BGEZAL
9135           if(opcode2[i-1]&0x10) { // BxxZAL
9136             alloc_reg(&branch_regs[i-1],i-1,31);
9137             dirty_reg(&branch_regs[i-1],31);
9138             branch_regs[i-1].is32|=1LL<<31;
9139           }
9140           break;
9141         case FJUMP:
9142           if(likely[i-1]==0) // BC1F/BC1T
9143           {
9144             alloc_cc(&current,i-1);
9145             dirty_reg(&current,CCREG);
9146             if(itype[i]==FCOMP) {
9147               // The delay slot overwrote the branch condition
9148               // Delay slot goes after the test (in order)
9149               delayslot_alloc(&current,i);
9150               current.isconst=0;
9151             }
9152             else
9153             {
9154               current.u=branch_unneeded_reg[i-1]&~(1LL<<rs1[i-1]);
9155               current.uu=branch_unneeded_reg_upper[i-1]&~(1LL<<us1[i-1]);
9156               // Alloc the branch condition register
9157               alloc_reg(&current,i-1,FSREG);
9158             }
9159             memcpy(&branch_regs[i-1],&current,sizeof(current));
9160             memcpy(&branch_regs[i-1].regmap_entry,&current.regmap,sizeof(current.regmap));
9161           }
9162           else // BC1FL/BC1TL
9163           {
9164             // Alloc the delay slot in case the branch is taken
9165             memcpy(&branch_regs[i-1],&current,sizeof(current));
9166             branch_regs[i-1].u=(branch_unneeded_reg[i-1]&~((1LL<<rs1[i])|(1LL<<rs2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9167             branch_regs[i-1].uu=(branch_unneeded_reg_upper[i-1]&~((1LL<<us1[i])|(1LL<<us2[i])|(1LL<<rt1[i])|(1LL<<rt2[i])))|1;
9168             if((~branch_regs[i-1].uu>>rt1[i])&1) branch_regs[i-1].uu&=~((1LL<<dep1[i])|(1LL<<dep2[i]))|1;
9169             alloc_cc(&branch_regs[i-1],i);
9170             dirty_reg(&branch_regs[i-1],CCREG);
9171             delayslot_alloc(&branch_regs[i-1],i);
9172             branch_regs[i-1].isconst=0;
9173             alloc_reg(&current,i,CCREG); // Not taken path
9174             dirty_reg(&current,CCREG);
9175             memcpy(&branch_regs[i-1].regmap_entry,&branch_regs[i-1].regmap,sizeof(current.regmap));
9176           }
9177           break;
9178       }
9179
9180       if(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000)
9181       {
9182         if(rt1[i-1]==31) // JAL/JALR
9183         {
9184           // Subroutine call will return here, don't alloc any registers
9185           current.is32=1;
9186           current.dirty=0;
9187           clear_all_regs(current.regmap);
9188           alloc_reg(&current,i,CCREG);
9189           dirty_reg(&current,CCREG);
9190         }
9191         else if(i+1<slen)
9192         {
9193           // Internal branch will jump here, match registers to caller
9194           current.is32=0x3FFFFFFFFLL;
9195           current.dirty=0;
9196           clear_all_regs(current.regmap);
9197           alloc_reg(&current,i,CCREG);
9198           dirty_reg(&current,CCREG);
9199           for(j=i-1;j>=0;j--)
9200           {
9201             if(ba[j]==start+i*4+4) {
9202               memcpy(current.regmap,branch_regs[j].regmap,sizeof(current.regmap));
9203               current.is32=branch_regs[j].is32;
9204               current.dirty=branch_regs[j].dirty;
9205               break;
9206             }
9207           }
9208           while(j>=0) {
9209             if(ba[j]==start+i*4+4) {
9210               for(hr=0;hr<HOST_REGS;hr++) {
9211                 if(current.regmap[hr]!=branch_regs[j].regmap[hr]) {
9212                   current.regmap[hr]=-1;
9213                 }
9214                 current.is32&=branch_regs[j].is32;
9215                 current.dirty&=branch_regs[j].dirty;
9216               }
9217             }
9218             j--;
9219           }
9220         }
9221       }
9222     }
9223
9224     // Count cycles in between branches
9225     ccadj[i]=cc;
9226     if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP||itype[i]==SYSCALL))
9227     {
9228       cc=0;
9229     }
9230     else
9231     {
9232       cc++;
9233     }
9234
9235     flush_dirty_uppers(&current);
9236     if(!is_ds[i]) {
9237       regs[i].is32=current.is32;
9238       regs[i].dirty=current.dirty;
9239       regs[i].isconst=current.isconst;
9240       memcpy(constmap[i],current.constmap,sizeof(current.constmap));
9241     }
9242     for(hr=0;hr<HOST_REGS;hr++) {
9243       if(hr!=EXCLUDE_REG&&regs[i].regmap[hr]>=0) {
9244         if(regmap_pre[i][hr]!=regs[i].regmap[hr]) {
9245           regs[i].wasconst&=~(1<<hr);
9246         }
9247       }
9248     }
9249     if(current.regmap[HOST_BTREG]==BTREG) current.regmap[HOST_BTREG]=-1;
9250   }
9251
9252   /* Pass 4 - Cull unused host registers */
9253
9254   uint64_t nr=0;
9255
9256   for (i=slen-1;i>=0;i--)
9257   {
9258     int hr;
9259     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
9260     {
9261       if(ba[i]<start || ba[i]>=(start+slen*4))
9262       {
9263         // Branch out of this block, don't need anything
9264         nr=0;
9265       }
9266       else
9267       {
9268         // Internal branch
9269         // Need whatever matches the target
9270         nr=0;
9271         int t=(ba[i]-start)>>2;
9272         for(hr=0;hr<HOST_REGS;hr++)
9273         {
9274           if(regs[i].regmap_entry[hr]>=0) {
9275             if(regs[i].regmap_entry[hr]==regs[t].regmap_entry[hr]) nr|=1<<hr;
9276           }
9277         }
9278       }
9279       // Conditional branch may need registers for following instructions
9280       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
9281       {
9282         if(i<slen-2) {
9283           nr|=needed_reg[i+2];
9284           for(hr=0;hr<HOST_REGS;hr++)
9285           {
9286             if(regmap_pre[i+2][hr]>=0&&get_reg(regs[i+2].regmap_entry,regmap_pre[i+2][hr])<0) nr&=~(1<<hr);
9287             //if((regmap_entry[i+2][hr])>=0) if(!((nr>>hr)&1)) DebugMessage(M64MSG_VERBOSE, "%x-bogus(%d=%d)",start+i*4,hr,regmap_entry[i+2][hr]);
9288           }
9289         }
9290       }
9291       // Don't need stuff which is overwritten
9292       if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
9293       if(regs[i].regmap[hr]<0) nr&=~(1<<hr);    //moved...
9294       // Merge in delay slot
9295       for(hr=0;hr<HOST_REGS;hr++)
9296       {
9297                 // Don't need stuff which is overwritten
9298 /*              if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr); //*SEB* Moved here
9299                 if(regs[i].regmap[hr]<0) nr&=~(1<<hr);*/
9300
9301         if(!likely[i]) {
9302           // These are overwritten unless the branch is "likely"
9303           // and the delay slot is nullified if not taken
9304           if(rt1[i+1]&&rt1[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
9305           if(rt2[i+1]&&rt2[i+1]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
9306         }
9307         if(us1[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9308         if(us2[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9309         if(rs1[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
9310         if(rs2[i+1]==regmap_pre[i][hr]) nr|=1<<hr;
9311         if(us1[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9312         if(us2[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9313         if(rs1[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
9314         if(rs2[i+1]==regs[i].regmap_entry[hr]) nr|=1<<hr;
9315         if(dep1[i+1]&&!((unneeded_reg_upper[i]>>dep1[i+1])&1)) {
9316           if(dep1[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9317           if(dep2[i+1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9318         }
9319         if(dep2[i+1]&&!((unneeded_reg_upper[i]>>dep2[i+1])&1)) {
9320           if(dep1[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9321           if(dep2[i+1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9322         }
9323         if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39) {
9324           if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
9325           if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
9326         }
9327       }
9328     }
9329     else if(itype[i]==SYSCALL)
9330     {
9331       // SYSCALL instruction (software interrupt)
9332       nr=0;
9333     }
9334     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
9335     {
9336       // ERET instruction (return from interrupt)
9337       nr=0;
9338     }
9339     else // Non-branch
9340     {
9341       if(i<slen-1) {
9342         for(hr=0;hr<HOST_REGS;hr++) {
9343           if(regmap_pre[i+1][hr]>=0&&get_reg(regs[i+1].regmap_entry,regmap_pre[i+1][hr])<0) nr&=~(1<<hr);
9344           if(regs[i].regmap[hr]!=regmap_pre[i+1][hr]) nr&=~(1<<hr);
9345           if(regs[i].regmap[hr]!=regmap_pre[i][hr]) nr&=~(1<<hr);
9346           if(regs[i].regmap[hr]<0) nr&=~(1<<hr);
9347         }
9348       }
9349     }
9350     for(hr=0;hr<HOST_REGS;hr++)
9351     {
9352       // Overwritten registers are not needed
9353       if(rt1[i]&&rt1[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
9354       if(rt2[i]&&rt2[i]==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
9355       if(FTEMP==(regs[i].regmap[hr]&63)) nr&=~(1<<hr);
9356       // Source registers are needed
9357       if(us1[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9358       if(us2[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9359       if(rs1[i]==regmap_pre[i][hr]) nr|=1<<hr;
9360       if(rs2[i]==regmap_pre[i][hr]) nr|=1<<hr;
9361       if(us1[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9362       if(us2[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9363       if(rs1[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
9364       if(rs2[i]==regs[i].regmap_entry[hr]) nr|=1<<hr;
9365       if(dep1[i]&&!((unneeded_reg_upper[i]>>dep1[i])&1)) {
9366         if(dep1[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9367         if(dep1[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9368       }
9369       if(dep2[i]&&!((unneeded_reg_upper[i]>>dep2[i])&1)) {
9370         if(dep2[i]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9371         if(dep2[i]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9372       }
9373       if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39) {
9374         if(regmap_pre[i][hr]==INVCP) nr|=1<<hr;
9375         if(regs[i].regmap_entry[hr]==INVCP) nr|=1<<hr;
9376       }
9377       // Don't store a register immediately after writing it,
9378       // may prevent dual-issue.
9379       // But do so if this is a branch target, otherwise we
9380       // might have to load the register before the branch.
9381       if(i>0&&!bt[i]&&((regs[i].wasdirty>>hr)&1)) {
9382         if((regmap_pre[i][hr]>0&&regmap_pre[i][hr]<64&&!((unneeded_reg[i]>>regmap_pre[i][hr])&1)) ||
9383            (regmap_pre[i][hr]>64&&!((unneeded_reg_upper[i]>>(regmap_pre[i][hr]&63))&1)) ) {
9384           if(rt1[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9385           if(rt2[i-1]==(regmap_pre[i][hr]&63)) nr|=1<<hr;
9386         }
9387         if((regs[i].regmap_entry[hr]>0&&regs[i].regmap_entry[hr]<64&&!((unneeded_reg[i]>>regs[i].regmap_entry[hr])&1)) ||
9388            (regs[i].regmap_entry[hr]>64&&!((unneeded_reg_upper[i]>>(regs[i].regmap_entry[hr]&63))&1)) ) {
9389           if(rt1[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9390           if(rt2[i-1]==(regs[i].regmap_entry[hr]&63)) nr|=1<<hr;
9391         }
9392       }
9393     }
9394     // Cycle count is needed at branches.  Assume it is needed at the target too.
9395     if(i==0||bt[i]||itype[i]==CJUMP||itype[i]==FJUMP||itype[i]==SPAN) {
9396       if(regmap_pre[i][HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
9397       if(regs[i].regmap_entry[HOST_CCREG]==CCREG) nr|=1<<HOST_CCREG;
9398     }
9399     // Save it
9400     needed_reg[i]=nr;
9401
9402     // Deallocate unneeded registers
9403     for(hr=0;hr<HOST_REGS;hr++)
9404     {
9405       if(!((nr>>hr)&1)) {
9406         if(regs[i].regmap_entry[hr]!=CCREG) regs[i].regmap_entry[hr]=-1;
9407         if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
9408            (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
9409            (regs[i].regmap[hr]&63)!=PTEMP && (regs[i].regmap[hr]&63)!=CCREG)
9410         {
9411           if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
9412           {
9413             if(likely[i]) {
9414               regs[i].regmap[hr]=-1;
9415               regs[i].isconst&=~(1<<hr);
9416               if(i<slen-2) {
9417                 regmap_pre[i+2][hr]=-1;
9418                 regs[i+2].wasconst&=~(1<<hr);
9419               }
9420             }
9421           }
9422         }
9423         if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
9424         {
9425           int d1=0,d2=0,map=0,temp=0;
9426           if(get_reg(regs[i].regmap,rt1[i+1]|64)>=0||get_reg(branch_regs[i].regmap,rt1[i+1]|64)>=0)
9427           {
9428             d1=dep1[i+1];
9429             d2=dep2[i+1];
9430           }
9431           if(using_tlb) {
9432             if(itype[i+1]==LOAD || itype[i+1]==LOADLR ||
9433                itype[i+1]==STORE || itype[i+1]==STORELR ||
9434                itype[i+1]==C1LS )
9435             map=TLREG;
9436           } else
9437           if(itype[i+1]==STORE || itype[i+1]==STORELR || (opcode[i+1]&0x3b)==0x39) {
9438             map=INVCP;
9439           }
9440           if(itype[i+1]==LOADLR || itype[i+1]==STORELR ||
9441              itype[i+1]==C1LS )
9442             temp=FTEMP;
9443           if((regs[i].regmap[hr]&63)!=rs1[i] && (regs[i].regmap[hr]&63)!=rs2[i] &&
9444              (regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
9445              (regs[i].regmap[hr]&63)!=rt1[i+1] && (regs[i].regmap[hr]&63)!=rt2[i+1] &&
9446              (regs[i].regmap[hr]^64)!=us1[i+1] && (regs[i].regmap[hr]^64)!=us2[i+1] &&
9447              (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
9448              regs[i].regmap[hr]!=rs1[i+1] && regs[i].regmap[hr]!=rs2[i+1] &&
9449              (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=PTEMP &&
9450              regs[i].regmap[hr]!=RHASH && regs[i].regmap[hr]!=RHTBL &&
9451              regs[i].regmap[hr]!=RTEMP && regs[i].regmap[hr]!=CCREG &&
9452              regs[i].regmap[hr]!=map )
9453           {
9454             regs[i].regmap[hr]=-1;
9455             regs[i].isconst&=~(1<<hr);
9456             if((branch_regs[i].regmap[hr]&63)!=rs1[i] && (branch_regs[i].regmap[hr]&63)!=rs2[i] &&
9457                (branch_regs[i].regmap[hr]&63)!=rt1[i] && (branch_regs[i].regmap[hr]&63)!=rt2[i] &&
9458                (branch_regs[i].regmap[hr]&63)!=rt1[i+1] && (branch_regs[i].regmap[hr]&63)!=rt2[i+1] &&
9459                (branch_regs[i].regmap[hr]^64)!=us1[i+1] && (branch_regs[i].regmap[hr]^64)!=us2[i+1] &&
9460                (branch_regs[i].regmap[hr]^64)!=d1 && (branch_regs[i].regmap[hr]^64)!=d2 &&
9461                branch_regs[i].regmap[hr]!=rs1[i+1] && branch_regs[i].regmap[hr]!=rs2[i+1] &&
9462                (branch_regs[i].regmap[hr]&63)!=temp && branch_regs[i].regmap[hr]!=PTEMP &&
9463                branch_regs[i].regmap[hr]!=RHASH && branch_regs[i].regmap[hr]!=RHTBL &&
9464                branch_regs[i].regmap[hr]!=RTEMP && branch_regs[i].regmap[hr]!=CCREG &&
9465                branch_regs[i].regmap[hr]!=map)
9466             {
9467               branch_regs[i].regmap[hr]=-1;
9468               branch_regs[i].regmap_entry[hr]=-1;
9469               if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
9470               {
9471                 if(!likely[i]&&i<slen-2) {
9472                   regmap_pre[i+2][hr]=-1;
9473                   regs[i+2].wasconst&=~(1<<hr);
9474                 }
9475               }
9476             }
9477           }
9478         }
9479         else
9480         {
9481           // Non-branch
9482           if(i>0)
9483           {
9484             int d1=0,d2=0,map=-1,temp=-1;
9485             if(get_reg(regs[i].regmap,rt1[i]|64)>=0)
9486             {
9487               d1=dep1[i];
9488               d2=dep2[i];
9489             }
9490             if(using_tlb) {
9491               if(itype[i]==LOAD || itype[i]==LOADLR ||
9492                  itype[i]==STORE || itype[i]==STORELR ||
9493                  itype[i]==C1LS )
9494               map=TLREG;
9495             } else if(itype[i]==STORE || itype[i]==STORELR || (opcode[i]&0x3b)==0x39) {
9496               map=INVCP;
9497             }
9498             if(itype[i]==LOADLR || itype[i]==STORELR ||
9499                itype[i]==C1LS )
9500               temp=FTEMP;
9501             if((regs[i].regmap[hr]&63)!=rt1[i] && (regs[i].regmap[hr]&63)!=rt2[i] &&
9502                (regs[i].regmap[hr]^64)!=us1[i] && (regs[i].regmap[hr]^64)!=us2[i] &&
9503                (regs[i].regmap[hr]^64)!=d1 && (regs[i].regmap[hr]^64)!=d2 &&
9504                regs[i].regmap[hr]!=rs1[i] && regs[i].regmap[hr]!=rs2[i] &&
9505                (regs[i].regmap[hr]&63)!=temp && regs[i].regmap[hr]!=map &&
9506                (itype[i]!=SPAN||regs[i].regmap[hr]!=CCREG))
9507             {
9508               if(i<slen-1&&!is_ds[i]) {
9509                 if(regmap_pre[i+1][hr]!=-1 || regs[i].regmap[hr]!=-1)
9510                 if(regmap_pre[i+1][hr]!=regs[i].regmap[hr])
9511                 if(regs[i].regmap[hr]<64||!((regs[i].was32>>(regs[i].regmap[hr]&63))&1))
9512                 {
9513                   DebugMessage(M64MSG_VERBOSE, "fail: %x (%d %d!=%d)",start+i*4,hr,regmap_pre[i+1][hr],regs[i].regmap[hr]);
9514                   assert(regmap_pre[i+1][hr]==regs[i].regmap[hr]);
9515                 }
9516                 regmap_pre[i+1][hr]=-1;
9517                 if(regs[i+1].regmap_entry[hr]==CCREG) regs[i+1].regmap_entry[hr]=-1;
9518                 regs[i+1].wasconst&=~(1<<hr);
9519               }
9520               regs[i].regmap[hr]=-1;
9521               regs[i].isconst&=~(1<<hr);
9522             }
9523           }
9524         }
9525       }
9526     }
9527   }
9528
9529   /* Pass 5 - Pre-allocate registers */
9530
9531   // If a register is allocated during a loop, try to allocate it for the
9532   // entire loop, if possible.  This avoids loading/storing registers
9533   // inside of the loop.
9534
9535   signed char f_regmap[HOST_REGS];
9536   clear_all_regs(f_regmap);
9537   for(i=0;i<slen-1;i++)
9538   {
9539     if(itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
9540     {
9541       if(ba[i]>=start && ba[i]<(start+i*4))
9542       if(itype[i+1]==NOP||itype[i+1]==MOV||itype[i+1]==ALU
9543       ||itype[i+1]==SHIFTIMM||itype[i+1]==IMM16||itype[i+1]==LOAD
9544       ||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS
9545       ||itype[i+1]==SHIFT||itype[i+1]==COP1||itype[i+1]==FLOAT
9546       ||itype[i+1]==FCOMP||itype[i+1]==FCONV)
9547       {
9548         int t=(ba[i]-start)>>2;
9549         if(t>0&&(itype[t-1]!=UJUMP&&itype[t-1]!=RJUMP&&itype[t-1]!=CJUMP&&itype[t-1]!=SJUMP&&itype[t-1]!=FJUMP)) // loop_preload can't handle jumps into delay slots
9550         if(t<2||(itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||rt1[t-2]!=31) // call/ret assumes no registers allocated
9551         for(hr=0;hr<HOST_REGS;hr++)
9552         {
9553           if(regs[i].regmap[hr]>64) {
9554             if(!((regs[i].dirty>>hr)&1))
9555               f_regmap[hr]=regs[i].regmap[hr];
9556             else f_regmap[hr]=-1;
9557           }
9558           else if(regs[i].regmap[hr]>=0) {
9559             if(f_regmap[hr]!=regs[i].regmap[hr]) {
9560               // dealloc old register
9561               int n;
9562               for(n=0;n<HOST_REGS;n++)
9563               {
9564                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
9565               }
9566               // and alloc new one
9567               f_regmap[hr]=regs[i].regmap[hr];
9568             }
9569           }
9570           if(branch_regs[i].regmap[hr]>64) {
9571             if(!((branch_regs[i].dirty>>hr)&1))
9572               f_regmap[hr]=branch_regs[i].regmap[hr];
9573             else f_regmap[hr]=-1;
9574           }
9575           else if(branch_regs[i].regmap[hr]>=0) {
9576             if(f_regmap[hr]!=branch_regs[i].regmap[hr]) {
9577               // dealloc old register
9578               int n;
9579               for(n=0;n<HOST_REGS;n++)
9580               {
9581                 if(f_regmap[n]==branch_regs[i].regmap[hr]) {f_regmap[n]=-1;}
9582               }
9583               // and alloc new one
9584               f_regmap[hr]=branch_regs[i].regmap[hr];
9585             }
9586           }
9587           if(ooo[i]) {
9588             if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1])
9589               f_regmap[hr]=branch_regs[i].regmap[hr];
9590           }else{
9591             if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1])
9592               f_regmap[hr]=branch_regs[i].regmap[hr];
9593           }
9594           // Avoid dirty->clean transition
9595           #ifdef DESTRUCTIVE_WRITEBACK
9596           if(t>0) if(get_reg(regmap_pre[t],f_regmap[hr])>=0) if((regs[t].wasdirty>>get_reg(regmap_pre[t],f_regmap[hr]))&1) f_regmap[hr]=-1;
9597           #endif
9598           // This check is only strictly required in the DESTRUCTIVE_WRITEBACK
9599           // case above, however it's always a good idea.  We can't hoist the
9600           // load if the register was already allocated, so there's no point
9601           // wasting time analyzing most of these cases.  It only "succeeds"
9602           // when the mapping was different and the load can be replaced with
9603           // a mov, which is of negligible benefit.  So such cases are
9604           // skipped below.
9605           if(f_regmap[hr]>0) {
9606             if(regs[t].regmap[hr]==f_regmap[hr]||(regs[t].regmap_entry[hr]<0&&get_reg(regmap_pre[t],f_regmap[hr])<0)) {
9607               int r=f_regmap[hr];
9608               for(j=t;j<=i;j++)
9609               {
9610                 //DebugMessage(M64MSG_VERBOSE, "Test %x -> %x, %x %d/%d",start+i*4,ba[i],start+j*4,hr,r);
9611                 if(r<34&&((unneeded_reg[j]>>r)&1)) break;
9612                 if(r>63&&((unneeded_reg_upper[j]>>(r&63))&1)) break;
9613                 if(r>63) {
9614                   // NB This can exclude the case where the upper-half
9615                   // register is lower numbered than the lower-half
9616                   // register.  Not sure if it's worth fixing...
9617                   if(get_reg(regs[j].regmap,r&63)<0) break;
9618                   if(get_reg(regs[j].regmap_entry,r&63)<0) break;
9619                   if(regs[j].is32&(1LL<<(r&63))) break;
9620                 }
9621                 if(regs[j].regmap[hr]==f_regmap[hr]&&(f_regmap[hr]&63)<TEMPREG) {
9622                   //DebugMessage(M64MSG_VERBOSE, "Hit %x -> %x, %x %d/%d",start+i*4,ba[i],start+j*4,hr,r);
9623                   int k;
9624                   if(regs[i].regmap[hr]==-1&&branch_regs[i].regmap[hr]==-1) {
9625                     if(get_reg(regs[i+2].regmap,f_regmap[hr])>=0) break;
9626                     if(r>63) {
9627                       if(get_reg(regs[i].regmap,r&63)<0) break;
9628                       if(get_reg(branch_regs[i].regmap,r&63)<0) break;
9629                     }
9630                     k=i;
9631                     while(k>1&&regs[k-1].regmap[hr]==-1) {
9632                       if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
9633                         //DebugMessage(M64MSG_VERBOSE, "no free regs for store %x",start+(k-1)*4);
9634                         break;
9635                       }
9636                       if(get_reg(regs[k-1].regmap,f_regmap[hr])>=0) {
9637                         //DebugMessage(M64MSG_VERBOSE, "no-match due to different register");
9638                         break;
9639                       }
9640                       if(itype[k-2]==UJUMP||itype[k-2]==RJUMP||itype[k-2]==CJUMP||itype[k-2]==SJUMP||itype[k-2]==FJUMP) {
9641                         //DebugMessage(M64MSG_VERBOSE, "no-match due to branch");
9642                         break;
9643                       }
9644                       // call/ret fast path assumes no registers allocated
9645                       if(k>2&&(itype[k-3]==UJUMP||itype[k-3]==RJUMP)&&rt1[k-3]==31) {
9646                         break;
9647                       }
9648                       if(r>63) {
9649                         // NB This can exclude the case where the upper-half
9650                         // register is lower numbered than the lower-half
9651                         // register.  Not sure if it's worth fixing...
9652                         if(get_reg(regs[k-1].regmap,r&63)<0) break;
9653                         if(regs[k-1].is32&(1LL<<(r&63))) break;
9654                       }
9655                       k--;
9656                     }
9657                     if(i<slen-1) {
9658                       if((regs[k].is32&(1LL<<f_regmap[hr]))!=
9659                         (regs[i+2].was32&(1LL<<f_regmap[hr]))) {
9660                         //DebugMessage(M64MSG_VERBOSE, "bad match after branch");
9661                         break;
9662                       }
9663                     }
9664                     if(regs[k-1].regmap[hr]==f_regmap[hr]&&regmap_pre[k][hr]==f_regmap[hr]) {
9665                       //DebugMessage(M64MSG_VERBOSE, "Extend r%d, %x ->",hr,start+k*4);
9666                       while(k<i) {
9667                         regs[k].regmap_entry[hr]=f_regmap[hr];
9668                         regs[k].regmap[hr]=f_regmap[hr];
9669                         regmap_pre[k+1][hr]=f_regmap[hr];
9670                         regs[k].wasdirty&=~(1<<hr);
9671                         regs[k].dirty&=~(1<<hr);
9672                         regs[k].wasdirty|=(1<<hr)&regs[k-1].dirty;
9673                         regs[k].dirty|=(1<<hr)&regs[k].wasdirty;
9674                         regs[k].wasconst&=~(1<<hr);
9675                         regs[k].isconst&=~(1<<hr);
9676                         k++;
9677                       }
9678                     }
9679                     else {
9680                       //DebugMessage(M64MSG_VERBOSE, "Fail Extend r%d, %x ->",hr,start+k*4);
9681                       break;
9682                     }
9683                     assert(regs[i-1].regmap[hr]==f_regmap[hr]);
9684                     if(regs[i-1].regmap[hr]==f_regmap[hr]&&regmap_pre[i][hr]==f_regmap[hr]) {
9685                       //DebugMessage(M64MSG_VERBOSE, "OK fill %x (r%d)",start+i*4,hr);
9686                       regs[i].regmap_entry[hr]=f_regmap[hr];
9687                       regs[i].regmap[hr]=f_regmap[hr];
9688                       regs[i].wasdirty&=~(1<<hr);
9689                       regs[i].dirty&=~(1<<hr);
9690                       regs[i].wasdirty|=(1<<hr)&regs[i-1].dirty;
9691                       regs[i].dirty|=(1<<hr)&regs[i-1].dirty;
9692                       regs[i].wasconst&=~(1<<hr);
9693                       regs[i].isconst&=~(1<<hr);
9694                       branch_regs[i].regmap_entry[hr]=f_regmap[hr];
9695                       branch_regs[i].wasdirty&=~(1<<hr);
9696                       branch_regs[i].wasdirty|=(1<<hr)&regs[i].dirty;
9697                       branch_regs[i].regmap[hr]=f_regmap[hr];
9698                       branch_regs[i].dirty&=~(1<<hr);
9699                       branch_regs[i].dirty|=(1<<hr)&regs[i].dirty;
9700                       branch_regs[i].wasconst&=~(1<<hr);
9701                       branch_regs[i].isconst&=~(1<<hr);
9702                       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000) {
9703                         regmap_pre[i+2][hr]=f_regmap[hr];
9704                         regs[i+2].wasdirty&=~(1<<hr);
9705                         regs[i+2].wasdirty|=(1<<hr)&regs[i].dirty;
9706                         assert((branch_regs[i].is32&(1LL<<f_regmap[hr]))==
9707                           (regs[i+2].was32&(1LL<<f_regmap[hr])));
9708                       }
9709                     }
9710                   }
9711                   for(k=t;k<j;k++) {
9712                     // Alloc register clean at beginning of loop,
9713                     // but may dirty it in pass 6
9714                     regs[k].regmap_entry[hr]=f_regmap[hr];
9715                     regs[k].regmap[hr]=f_regmap[hr];
9716                     regs[k].dirty&=~(1<<hr);
9717                     regs[k].wasconst&=~(1<<hr);
9718                     regs[k].isconst&=~(1<<hr);
9719                     if(itype[k]==UJUMP||itype[k]==RJUMP||itype[k]==CJUMP||itype[k]==SJUMP||itype[k]==FJUMP) {
9720                       branch_regs[k].regmap_entry[hr]=f_regmap[hr];
9721                       branch_regs[k].regmap[hr]=f_regmap[hr];
9722                       branch_regs[k].dirty&=~(1<<hr);
9723                       branch_regs[k].wasconst&=~(1<<hr);
9724                       branch_regs[k].isconst&=~(1<<hr);
9725                       if(itype[k]!=RJUMP&&itype[k]!=UJUMP&&(source[k]>>16)!=0x1000) {
9726                         regmap_pre[k+2][hr]=f_regmap[hr];
9727                         regs[k+2].wasdirty&=~(1<<hr);
9728                         assert((branch_regs[k].is32&(1LL<<f_regmap[hr]))==
9729                           (regs[k+2].was32&(1LL<<f_regmap[hr])));
9730                       }
9731                     }
9732                     else
9733                     {
9734                       regmap_pre[k+1][hr]=f_regmap[hr];
9735                       regs[k+1].wasdirty&=~(1<<hr);
9736                     }
9737                   }
9738                   if(regs[j].regmap[hr]==f_regmap[hr])
9739                     regs[j].regmap_entry[hr]=f_regmap[hr];
9740                   break;
9741                 }
9742                 if(j==i) break;
9743                 if(regs[j].regmap[hr]>=0)
9744                   break;
9745                 if(get_reg(regs[j].regmap,f_regmap[hr])>=0) {
9746                   //DebugMessage(M64MSG_VERBOSE, "no-match due to different register");
9747                   break;
9748                 }
9749                 if((regs[j+1].is32&(1LL<<f_regmap[hr]))!=(regs[j].is32&(1LL<<f_regmap[hr]))) {
9750                   //DebugMessage(M64MSG_VERBOSE, "32/64 mismatch %x %d",start+j*4,hr);
9751                   break;
9752                 }
9753                 if(itype[j]==UJUMP||itype[j]==RJUMP||(source[j]>>16)==0x1000)
9754                 {
9755                   // Stop on unconditional branch
9756                   break;
9757                 }
9758                 if(itype[j]==CJUMP||itype[j]==SJUMP||itype[j]==FJUMP)
9759                 {
9760                   if(ooo[j]) {
9761                     if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1])
9762                       break;
9763                   }else{
9764                     if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1])
9765                       break;
9766                   }
9767                   if(get_reg(branch_regs[j].regmap,f_regmap[hr])>=0) {
9768                     //DebugMessage(M64MSG_VERBOSE, "no-match due to different register (branch)");
9769                     break;
9770                   }
9771                 }
9772                 if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
9773                   //DebugMessage(M64MSG_VERBOSE, "No free regs for store %x",start+j*4);
9774                   break;
9775                 }
9776                 if(f_regmap[hr]>=64) {
9777                   if(regs[j].is32&(1LL<<(f_regmap[hr]&63))) {
9778                     break;
9779                   }
9780                   else
9781                   {
9782                     if(get_reg(regs[j].regmap,f_regmap[hr]&63)<0) {
9783                       break;
9784                     }
9785                   }
9786                 }
9787               }
9788             }
9789           }
9790         }
9791       }
9792     }else{
9793       // Non branch or undetermined branch target
9794       for(hr=0;hr<HOST_REGS;hr++)
9795       {
9796         if(hr!=EXCLUDE_REG) {
9797           if(regs[i].regmap[hr]>64) {
9798             if(!((regs[i].dirty>>hr)&1))
9799               f_regmap[hr]=regs[i].regmap[hr];
9800           }
9801           else if(regs[i].regmap[hr]>=0) {
9802             if(f_regmap[hr]!=regs[i].regmap[hr]) {
9803               // dealloc old register
9804               int n;
9805               for(n=0;n<HOST_REGS;n++)
9806               {
9807                 if(f_regmap[n]==regs[i].regmap[hr]) {f_regmap[n]=-1;}
9808               }
9809               // and alloc new one
9810               f_regmap[hr]=regs[i].regmap[hr];
9811             }
9812           }
9813         }
9814       }
9815       // Try to restore cycle count at branch targets
9816       if(bt[i]) {
9817         for(j=i;j<slen-1;j++) {
9818           if(regs[j].regmap[HOST_CCREG]!=-1) break;
9819           if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) {
9820             //DebugMessage(M64MSG_VERBOSE, "no free regs for store %x",start+j*4);
9821             break;
9822           }
9823         }
9824         if(regs[j].regmap[HOST_CCREG]==CCREG) {
9825           int k=i;
9826           //DebugMessage(M64MSG_VERBOSE, "Extend CC, %x -> %x",start+k*4,start+j*4);
9827           while(k<j) {
9828             regs[k].regmap_entry[HOST_CCREG]=CCREG;
9829             regs[k].regmap[HOST_CCREG]=CCREG;
9830             regmap_pre[k+1][HOST_CCREG]=CCREG;
9831             regs[k+1].wasdirty|=1<<HOST_CCREG;
9832             regs[k].dirty|=1<<HOST_CCREG;
9833             regs[k].wasconst&=~(1<<HOST_CCREG);
9834             regs[k].isconst&=~(1<<HOST_CCREG);
9835             k++;
9836           }
9837           regs[j].regmap_entry[HOST_CCREG]=CCREG;
9838         }
9839         // Work backwards from the branch target
9840         if(j>i&&f_regmap[HOST_CCREG]==CCREG)
9841         {
9842           //DebugMessage(M64MSG_VERBOSE, "Extend backwards");
9843           int k;
9844           k=i;
9845           while(regs[k-1].regmap[HOST_CCREG]==-1) {
9846             if(count_free_regs(regs[k-1].regmap)<=minimum_free_regs[k-1]) {
9847               //DebugMessage(M64MSG_VERBOSE, "no free regs for store %x",start+(k-1)*4);
9848               break;
9849             }
9850             k--;
9851           }
9852           if(regs[k-1].regmap[HOST_CCREG]==CCREG) {
9853             //DebugMessage(M64MSG_VERBOSE, "Extend CC, %x ->",start+k*4);
9854             while(k<=i) {
9855               regs[k].regmap_entry[HOST_CCREG]=CCREG;
9856               regs[k].regmap[HOST_CCREG]=CCREG;
9857               regmap_pre[k+1][HOST_CCREG]=CCREG;
9858               regs[k+1].wasdirty|=1<<HOST_CCREG;
9859               regs[k].dirty|=1<<HOST_CCREG;
9860               regs[k].wasconst&=~(1<<HOST_CCREG);
9861               regs[k].isconst&=~(1<<HOST_CCREG);
9862               k++;
9863             }
9864           }
9865           else {
9866             //DebugMessage(M64MSG_VERBOSE, "Fail Extend CC, %x ->",start+k*4);
9867           }
9868         }
9869       }
9870       if(itype[i]!=STORE&&itype[i]!=STORELR&&itype[i]!=C1LS&&itype[i]!=SHIFT&&
9871          itype[i]!=NOP&&itype[i]!=MOV&&itype[i]!=ALU&&itype[i]!=SHIFTIMM&&
9872          itype[i]!=IMM16&&itype[i]!=LOAD&&itype[i]!=COP1&&itype[i]!=FLOAT&&
9873          itype[i]!=FCONV&&itype[i]!=FCOMP)
9874       {
9875         memcpy(f_regmap,regs[i].regmap,sizeof(f_regmap));
9876       }
9877     }
9878   }
9879
9880   // Cache memory offset or tlb map pointer if a register is available
9881   #ifndef HOST_IMM_ADDR32
9882   #ifndef RAM_OFFSET
9883   if(using_tlb)
9884   #endif
9885   {
9886     int earliest_available[HOST_REGS];
9887     int loop_start[HOST_REGS];
9888     int score[HOST_REGS];
9889     int end[HOST_REGS];
9890     int reg=using_tlb?MMREG:ROREG;
9891
9892     // Init
9893     for(hr=0;hr<HOST_REGS;hr++) {
9894       score[hr]=0;earliest_available[hr]=0;
9895       loop_start[hr]=MAXBLOCK;
9896     }
9897     for(i=0;i<slen-1;i++)
9898     {
9899       // Can't do anything if no registers are available
9900       if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i]) {
9901         for(hr=0;hr<HOST_REGS;hr++) {
9902           score[hr]=0;earliest_available[hr]=i+1;
9903           loop_start[hr]=MAXBLOCK;
9904         }
9905       }
9906       if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP) {
9907         if(!ooo[i]) {
9908           if(count_free_regs(branch_regs[i].regmap)<=minimum_free_regs[i+1]) {
9909             for(hr=0;hr<HOST_REGS;hr++) {
9910               score[hr]=0;earliest_available[hr]=i+1;
9911               loop_start[hr]=MAXBLOCK;
9912             }
9913           }
9914         }else{
9915           if(count_free_regs(regs[i].regmap)<=minimum_free_regs[i+1]) {
9916             for(hr=0;hr<HOST_REGS;hr++) {
9917               score[hr]=0;earliest_available[hr]=i+1;
9918               loop_start[hr]=MAXBLOCK;
9919             }
9920           }
9921         }
9922       }
9923       // Mark unavailable registers
9924       for(hr=0;hr<HOST_REGS;hr++) {
9925         if(regs[i].regmap[hr]>=0) {
9926           score[hr]=0;earliest_available[hr]=i+1;
9927           loop_start[hr]=MAXBLOCK;
9928         }
9929         if(itype[i]==UJUMP||itype[i]==RJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP) {
9930           if(branch_regs[i].regmap[hr]>=0) {
9931             score[hr]=0;earliest_available[hr]=i+2;
9932             loop_start[hr]=MAXBLOCK;
9933           }
9934         }
9935       }
9936       // No register allocations after unconditional jumps
9937       if(itype[i]==UJUMP||itype[i]==RJUMP||(source[i]>>16)==0x1000)
9938       {
9939         for(hr=0;hr<HOST_REGS;hr++) {
9940           score[hr]=0;earliest_available[hr]=i+2;
9941           loop_start[hr]=MAXBLOCK;
9942         }
9943         i++; // Skip delay slot too
9944         //DebugMessage(M64MSG_VERBOSE, "skip delay slot: %x",start+i*4);
9945       }
9946       else
9947       // Possible match
9948       if(itype[i]==LOAD||itype[i]==LOADLR||
9949          itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS) {
9950         for(hr=0;hr<HOST_REGS;hr++) {
9951           if(hr!=EXCLUDE_REG) {
9952             end[hr]=i-1;
9953             for(j=i;j<slen-1;j++) {
9954               if(regs[j].regmap[hr]>=0) break;
9955               if(itype[j]==UJUMP||itype[j]==RJUMP||itype[j]==CJUMP||itype[j]==SJUMP||itype[j]==FJUMP) {
9956                 if(branch_regs[j].regmap[hr]>=0) break;
9957                 if(ooo[j]) {
9958                   if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j+1]) break;
9959                 }else{
9960                   if(count_free_regs(branch_regs[j].regmap)<=minimum_free_regs[j+1]) break;
9961                 }
9962               }
9963               else if(count_free_regs(regs[j].regmap)<=minimum_free_regs[j]) break;
9964               if(itype[j]==UJUMP||itype[j]==RJUMP||itype[j]==CJUMP||itype[j]==SJUMP||itype[j]==FJUMP) {
9965                 int t=(ba[j]-start)>>2;
9966                 if(t<j&&t>=earliest_available[hr]) {
9967                   if(t==1||(t>1&&itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||(t>1&&rt1[t-2]!=31)) { // call/ret assumes no registers allocated
9968                     // Score a point for hoisting loop invariant
9969                     if(t<loop_start[hr]) loop_start[hr]=t;
9970                     //DebugMessage(M64MSG_VERBOSE, "set loop_start: i=%x j=%x (%x)",start+i*4,start+j*4,start+t*4);
9971                     score[hr]++;
9972                     end[hr]=j;
9973                   }
9974                 }
9975                 else if(t<j) {
9976                   if(regs[t].regmap[hr]==reg) {
9977                     // Score a point if the branch target matches this register
9978                     score[hr]++;
9979                     end[hr]=j;
9980                   }
9981                 }
9982                 if(itype[j+1]==LOAD||itype[j+1]==LOADLR||
9983                    itype[j+1]==STORE||itype[j+1]==STORELR||itype[j+1]==C1LS) {
9984                   score[hr]++;
9985                   end[hr]=j;
9986                 }
9987               }
9988               if(itype[j]==UJUMP||itype[j]==RJUMP||(source[j]>>16)==0x1000)
9989               {
9990                 // Stop on unconditional branch
9991                 break;
9992               }
9993               else
9994               if(itype[j]==LOAD||itype[j]==LOADLR||
9995                  itype[j]==STORE||itype[j]==STORELR||itype[j]==C1LS) {
9996                 score[hr]++;
9997                 end[hr]=j;
9998               }
9999             }
10000           }
10001         }
10002         // Find highest score and allocate that register
10003         int maxscore=0;
10004         for(hr=0;hr<HOST_REGS;hr++) {
10005           if(hr!=EXCLUDE_REG) {
10006             if(score[hr]>score[maxscore]) {
10007               maxscore=hr;
10008               //DebugMessage(M64MSG_VERBOSE, "highest score: %d %d (%x->%x)",score[hr],hr,start+i*4,start+end[hr]*4);
10009             }
10010           }
10011         }
10012         if(score[maxscore]>1)
10013         {
10014           if(i<loop_start[maxscore]) loop_start[maxscore]=i;
10015           for(j=loop_start[maxscore];j<slen&&j<=end[maxscore];j++) {
10016             //if(regs[j].regmap[maxscore]>=0) {DebugMessage(M64MSG_ERROR, "oops: %x %x was %d=%d",loop_start[maxscore]*4+start,j*4+start,maxscore,regs[j].regmap[maxscore]);}
10017             assert(regs[j].regmap[maxscore]<0);
10018             if(j>loop_start[maxscore]) regs[j].regmap_entry[maxscore]=reg;
10019             regs[j].regmap[maxscore]=reg;
10020             regs[j].dirty&=~(1<<maxscore);
10021             regs[j].wasconst&=~(1<<maxscore);
10022             regs[j].isconst&=~(1<<maxscore);
10023             if(itype[j]==UJUMP||itype[j]==RJUMP||itype[j]==CJUMP||itype[j]==SJUMP||itype[j]==FJUMP) {
10024               branch_regs[j].regmap[maxscore]=reg;
10025               branch_regs[j].wasdirty&=~(1<<maxscore);
10026               branch_regs[j].dirty&=~(1<<maxscore);
10027               branch_regs[j].wasconst&=~(1<<maxscore);
10028               branch_regs[j].isconst&=~(1<<maxscore);
10029               if(itype[j]!=RJUMP&&itype[j]!=UJUMP&&(source[j]>>16)!=0x1000) {
10030                 regmap_pre[j+2][maxscore]=reg;
10031                 regs[j+2].wasdirty&=~(1<<maxscore);
10032               }
10033               // loop optimization (loop_preload)
10034               int t=(ba[j]-start)>>2;
10035               if(t==loop_start[maxscore]) {
10036                 if(t==1||(t>1&&itype[t-2]!=UJUMP&&itype[t-2]!=RJUMP)||(t>1&&rt1[t-2]!=31)) // call/ret assumes no registers allocated
10037                   regs[t].regmap_entry[maxscore]=reg;
10038               }
10039             }
10040             else
10041             {
10042               if(j<1||(itype[j-1]!=RJUMP&&itype[j-1]!=UJUMP&&itype[j-1]!=CJUMP&&itype[j-1]!=SJUMP&&itype[j-1]!=FJUMP)) {
10043                 regmap_pre[j+1][maxscore]=reg;
10044                 regs[j+1].wasdirty&=~(1<<maxscore);
10045               }
10046             }
10047           }
10048           i=j-1;
10049           if(itype[j-1]==RJUMP||itype[j-1]==UJUMP||itype[j-1]==CJUMP||itype[j-1]==SJUMP||itype[j-1]==FJUMP) i++; // skip delay slot
10050           for(hr=0;hr<HOST_REGS;hr++) {
10051             score[hr]=0;earliest_available[hr]=i+i;
10052             loop_start[hr]=MAXBLOCK;
10053           }
10054         }
10055       }
10056     }
10057   }
10058   #endif
10059
10060   // This allocates registers (if possible) one instruction prior
10061   // to use, which can avoid a load-use penalty on certain CPUs.
10062   for(i=0;i<slen-1;i++)
10063   {
10064     if(!i||(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP&&itype[i-1]!=FJUMP))
10065     {
10066       if(!bt[i+1])
10067       {
10068         if(itype[i]==ALU||itype[i]==MOV||itype[i]==LOAD||itype[i]==SHIFTIMM||itype[i]==IMM16||(itype[i]==COP1&&opcode2[i]<3))
10069         {
10070           if(rs1[i+1]) {
10071             if((hr=get_reg(regs[i+1].regmap,rs1[i+1]))>=0)
10072             {
10073               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10074               {
10075                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
10076                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
10077                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
10078                 regs[i].isconst&=~(1<<hr);
10079                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10080                 constmap[i][hr]=constmap[i+1][hr];
10081                 regs[i+1].wasdirty&=~(1<<hr);
10082                 regs[i].dirty&=~(1<<hr);
10083               }
10084             }
10085           }
10086           if(rs2[i+1]) {
10087             if((hr=get_reg(regs[i+1].regmap,rs2[i+1]))>=0)
10088             {
10089               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10090               {
10091                 regs[i].regmap[hr]=regs[i+1].regmap[hr];
10092                 regmap_pre[i+1][hr]=regs[i+1].regmap[hr];
10093                 regs[i+1].regmap_entry[hr]=regs[i+1].regmap[hr];
10094                 regs[i].isconst&=~(1<<hr);
10095                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10096                 constmap[i][hr]=constmap[i+1][hr];
10097                 regs[i+1].wasdirty&=~(1<<hr);
10098                 regs[i].dirty&=~(1<<hr);
10099               }
10100             }
10101           }
10102           // Preload target address for load instruction (non-constant)
10103           if(itype[i+1]==LOAD&&rs1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
10104             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
10105             {
10106               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10107               {
10108                 regs[i].regmap[hr]=rs1[i+1];
10109                 regmap_pre[i+1][hr]=rs1[i+1];
10110                 regs[i+1].regmap_entry[hr]=rs1[i+1];
10111                 regs[i].isconst&=~(1<<hr);
10112                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10113                 constmap[i][hr]=constmap[i+1][hr];
10114                 regs[i+1].wasdirty&=~(1<<hr);
10115                 regs[i].dirty&=~(1<<hr);
10116               }
10117             }
10118           }
10119           // Load source into target register
10120           if(lt1[i+1]&&get_reg(regs[i+1].regmap,rs1[i+1])<0) {
10121             if((hr=get_reg(regs[i+1].regmap,rt1[i+1]))>=0)
10122             {
10123               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10124               {
10125                 regs[i].regmap[hr]=rs1[i+1];
10126                 regmap_pre[i+1][hr]=rs1[i+1];
10127                 regs[i+1].regmap_entry[hr]=rs1[i+1];
10128                 regs[i].isconst&=~(1<<hr);
10129                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10130                 constmap[i][hr]=constmap[i+1][hr];
10131                 regs[i+1].wasdirty&=~(1<<hr);
10132                 regs[i].dirty&=~(1<<hr);
10133               }
10134             }
10135           }
10136           // Preload map address
10137           #ifndef HOST_IMM_ADDR32
10138           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR||itype[i+1]==C1LS) {
10139             hr=get_reg(regs[i+1].regmap,TLREG);
10140             if(hr>=0) {
10141               int sr=get_reg(regs[i+1].regmap,rs1[i+1]);
10142               if(sr>=0&&((regs[i+1].wasconst>>sr)&1)) {
10143                 int nr;
10144                 if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10145                 {
10146                   regs[i].regmap[hr]=MGEN1+((i+1)&1);
10147                   regmap_pre[i+1][hr]=MGEN1+((i+1)&1);
10148                   regs[i+1].regmap_entry[hr]=MGEN1+((i+1)&1);
10149                   regs[i].isconst&=~(1<<hr);
10150                   regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10151                   constmap[i][hr]=constmap[i+1][hr];
10152                   regs[i+1].wasdirty&=~(1<<hr);
10153                   regs[i].dirty&=~(1<<hr);
10154                 }
10155                 else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
10156                 {
10157                   // move it to another register
10158                   regs[i+1].regmap[hr]=-1;
10159                   regmap_pre[i+2][hr]=-1;
10160                   regs[i+1].regmap[nr]=TLREG;
10161                   regmap_pre[i+2][nr]=TLREG;
10162                   regs[i].regmap[nr]=MGEN1+((i+1)&1);
10163                   regmap_pre[i+1][nr]=MGEN1+((i+1)&1);
10164                   regs[i+1].regmap_entry[nr]=MGEN1+((i+1)&1);
10165                   regs[i].isconst&=~(1<<nr);
10166                   regs[i+1].isconst&=~(1<<nr);
10167                   regs[i].dirty&=~(1<<nr);
10168                   regs[i+1].wasdirty&=~(1<<nr);
10169                   regs[i+1].dirty&=~(1<<nr);
10170                   regs[i+2].wasdirty&=~(1<<nr);
10171                 }
10172               }
10173             }
10174           }
10175           #endif
10176           // Address for store instruction (non-constant)
10177           if(itype[i+1]==STORE||itype[i+1]==STORELR||opcode[i+1]==0x39||opcode[i+1]==0x3D) { // SB/SH/SW/SD/SWC1/SDC1
10178             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
10179               hr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1);
10180               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
10181               else {regs[i+1].regmap[hr]=AGEN1+((i+1)&1);regs[i+1].isconst&=~(1<<hr);}
10182               assert(hr>=0);
10183               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10184               {
10185                 regs[i].regmap[hr]=rs1[i+1];
10186                 regmap_pre[i+1][hr]=rs1[i+1];
10187                 regs[i+1].regmap_entry[hr]=rs1[i+1];
10188                 regs[i].isconst&=~(1<<hr);
10189                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10190                 constmap[i][hr]=constmap[i+1][hr];
10191                 regs[i+1].wasdirty&=~(1<<hr);
10192                 regs[i].dirty&=~(1<<hr);
10193               }
10194             }
10195           }
10196           if(itype[i+1]==LOADLR||opcode[i+1]==0x31||opcode[i+1]==0x35) { // LWC1/LDC1
10197             if(get_reg(regs[i+1].regmap,rs1[i+1])<0) {
10198               int nr;
10199               hr=get_reg(regs[i+1].regmap,FTEMP);
10200               assert(hr>=0);
10201               if(regs[i].regmap[hr]<0&&regs[i+1].regmap_entry[hr]<0)
10202               {
10203                 regs[i].regmap[hr]=rs1[i+1];
10204                 regmap_pre[i+1][hr]=rs1[i+1];
10205                 regs[i+1].regmap_entry[hr]=rs1[i+1];
10206                 regs[i].isconst&=~(1<<hr);
10207                 regs[i].isconst|=regs[i+1].isconst&(1<<hr);
10208                 constmap[i][hr]=constmap[i+1][hr];
10209                 regs[i+1].wasdirty&=~(1<<hr);
10210                 regs[i].dirty&=~(1<<hr);
10211               }
10212               else if((nr=get_reg2(regs[i].regmap,regs[i+1].regmap,-1))>=0)
10213               {
10214                 // move it to another register
10215                 regs[i+1].regmap[hr]=-1;
10216                 regmap_pre[i+2][hr]=-1;
10217                 regs[i+1].regmap[nr]=FTEMP;
10218                 regmap_pre[i+2][nr]=FTEMP;
10219                 regs[i].regmap[nr]=rs1[i+1];
10220                 regmap_pre[i+1][nr]=rs1[i+1];
10221                 regs[i+1].regmap_entry[nr]=rs1[i+1];
10222                 regs[i].isconst&=~(1<<nr);
10223                 regs[i+1].isconst&=~(1<<nr);
10224                 regs[i].dirty&=~(1<<nr);
10225                 regs[i+1].wasdirty&=~(1<<nr);
10226                 regs[i+1].dirty&=~(1<<nr);
10227                 regs[i+2].wasdirty&=~(1<<nr);
10228               }
10229             }
10230           }
10231           if(itype[i+1]==LOAD||itype[i+1]==LOADLR||itype[i+1]==STORE||itype[i+1]==STORELR/*||itype[i+1]==C1LS*/) {
10232             if(itype[i+1]==LOAD)
10233               hr=get_reg(regs[i+1].regmap,rt1[i+1]);
10234             if(itype[i+1]==LOADLR||opcode[i+1]==0x31||opcode[i+1]==0x35) // LWC1/LDC1
10235               hr=get_reg(regs[i+1].regmap,FTEMP);
10236             if(itype[i+1]==STORE||itype[i+1]==STORELR||opcode[i+1]==0x39||opcode[i+1]==0x3D) { // SWC1/SDC1
10237               hr=get_reg(regs[i+1].regmap,AGEN1+((i+1)&1));
10238               if(hr<0) hr=get_reg(regs[i+1].regmap,-1);
10239             }
10240             if(hr>=0&&regs[i].regmap[hr]<0) {
10241               int rs=get_reg(regs[i+1].regmap,rs1[i+1]);
10242               if(rs>=0&&((regs[i+1].wasconst>>rs)&1)) {
10243                 regs[i].regmap[hr]=AGEN1+((i+1)&1);
10244                 regmap_pre[i+1][hr]=AGEN1+((i+1)&1);
10245                 regs[i+1].regmap_entry[hr]=AGEN1+((i+1)&1);
10246                 regs[i].isconst&=~(1<<hr);
10247                 regs[i+1].wasdirty&=~(1<<hr);
10248                 regs[i].dirty&=~(1<<hr);
10249               }
10250             }
10251           }
10252         }
10253       }
10254     }
10255   }
10256
10257   /* Pass 6 - Optimize clean/dirty state */
10258   clean_registers(0,slen-1,1);
10259
10260   /* Pass 7 - Identify 32-bit registers */
10261
10262   provisional_r32();
10263
10264   u_int r32=0;
10265
10266   for (i=slen-1;i>=0;i--)
10267   {
10268     int hr;
10269     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
10270     {
10271       if(ba[i]<start || ba[i]>=(start+slen*4))
10272       {
10273         // Branch out of this block, don't need anything
10274         r32=0;
10275       }
10276       else
10277       {
10278         // Internal branch
10279         // Need whatever matches the target
10280         // (and doesn't get overwritten by the delay slot instruction)
10281         r32=0;
10282         int t=(ba[i]-start)>>2;
10283         if(ba[i]>start+i*4) {
10284           // Forward branch
10285           if(!(requires_32bit[t]&~regs[i].was32))
10286             r32|=requires_32bit[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
10287         }else{
10288           // Backward branch
10289           //if(!(regs[t].was32&~unneeded_reg_upper[t]&~regs[i].was32))
10290           //  r32|=regs[t].was32&~unneeded_reg_upper[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
10291           if(!(pr32[t]&~regs[i].was32))
10292             r32|=pr32[t]&(~(1LL<<rt1[i+1]))&(~(1LL<<rt2[i+1]));
10293         }
10294       }
10295       // Conditional branch may need registers for following instructions
10296       if(itype[i]!=RJUMP&&itype[i]!=UJUMP&&(source[i]>>16)!=0x1000)
10297       {
10298         if(i<slen-2) {
10299           r32|=requires_32bit[i+2];
10300           r32&=regs[i].was32;
10301           // Mark this address as a branch target since it may be called
10302           // upon return from interrupt
10303           bt[i+2]=1;
10304         }
10305       }
10306       // Merge in delay slot
10307       if(!likely[i]) {
10308         // These are overwritten unless the branch is "likely"
10309         // and the delay slot is nullified if not taken
10310         r32&=~(1LL<<rt1[i+1]);
10311         r32&=~(1LL<<rt2[i+1]);
10312       }
10313       // Assume these are needed (delay slot)
10314       if(us1[i+1]>0)
10315       {
10316         if((regs[i].was32>>us1[i+1])&1) r32|=1LL<<us1[i+1];
10317       }
10318       if(us2[i+1]>0)
10319       {
10320         if((regs[i].was32>>us2[i+1])&1) r32|=1LL<<us2[i+1];
10321       }
10322       if(dep1[i+1]&&!((unneeded_reg_upper[i]>>dep1[i+1])&1))
10323       {
10324         if((regs[i].was32>>dep1[i+1])&1) r32|=1LL<<dep1[i+1];
10325       }
10326       if(dep2[i+1]&&!((unneeded_reg_upper[i]>>dep2[i+1])&1))
10327       {
10328         if((regs[i].was32>>dep2[i+1])&1) r32|=1LL<<dep2[i+1];
10329       }
10330     }
10331     else if(itype[i]==SYSCALL)
10332     {
10333       // SYSCALL instruction (software interrupt)
10334       r32=0;
10335     }
10336     else if(itype[i]==COP0 && (source[i]&0x3f)==0x18)
10337     {
10338       // ERET instruction (return from interrupt)
10339       r32=0;
10340     }
10341     // Check 32 bits
10342     r32&=~(1LL<<rt1[i]);
10343     r32&=~(1LL<<rt2[i]);
10344     if(us1[i]>0)
10345     {
10346       if((regs[i].was32>>us1[i])&1) r32|=1LL<<us1[i];
10347     }
10348     if(us2[i]>0)
10349     {
10350       if((regs[i].was32>>us2[i])&1) r32|=1LL<<us2[i];
10351     }
10352     if(dep1[i]&&!((unneeded_reg_upper[i]>>dep1[i])&1))
10353     {
10354       if((regs[i].was32>>dep1[i])&1) r32|=1LL<<dep1[i];
10355     }
10356     if(dep2[i]&&!((unneeded_reg_upper[i]>>dep2[i])&1))
10357     {
10358       if((regs[i].was32>>dep2[i])&1) r32|=1LL<<dep2[i];
10359     }
10360     requires_32bit[i]=r32;
10361
10362     // Dirty registers which are 32-bit, require 32-bit input
10363     // as they will be written as 32-bit values
10364     for(hr=0;hr<HOST_REGS;hr++)
10365     {
10366       if(regs[i].regmap_entry[hr]>0&&regs[i].regmap_entry[hr]<64) {
10367         if((regs[i].was32>>regs[i].regmap_entry[hr])&(regs[i].wasdirty>>hr)&1) {
10368           if(!((unneeded_reg_upper[i]>>regs[i].regmap_entry[hr])&1))
10369           requires_32bit[i]|=1LL<<regs[i].regmap_entry[hr];
10370         }
10371       }
10372     }
10373     //requires_32bit[i]=is32[i]&~unneeded_reg_upper[i]; // DEBUG
10374   }
10375
10376   if(itype[slen-1]==SPAN) {
10377     bt[slen-1]=1; // Mark as a branch target so instruction can restart after exception
10378   }
10379
10380   /* Debug/disassembly */
10381 //  if((void*)assem_debug==(void*)printf)
10382 #if defined( ASSEM_DEBUG )
10383   for(i=0;i<slen;i++)
10384   {
10385     DebugMessage(M64MSG_VERBOSE, "U:");
10386     int r;
10387     for(r=1;r<=CCREG;r++) {
10388       if((unneeded_reg[i]>>r)&1) {
10389         if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10390         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10391         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10392       }
10393     }
10394     DebugMessage(M64MSG_VERBOSE, " UU:");
10395     for(r=1;r<=CCREG;r++) {
10396       if(((unneeded_reg_upper[i]&~unneeded_reg[i])>>r)&1) {
10397         if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10398         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10399         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10400       }
10401     }
10402     DebugMessage(M64MSG_VERBOSE, " 32:");
10403     for(r=0;r<=CCREG;r++) {
10404       //if(((is32[i]>>r)&(~unneeded_reg[i]>>r))&1) {
10405       if((regs[i].was32>>r)&1) {
10406         if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10407         else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10408         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10409         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10410       }
10411     }
10412     #if NEW_DYNAREC == NEW_DYNAREC_X86
10413     DebugMessage(M64MSG_VERBOSE, "pre: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7]);
10414     #endif
10415     #if NEW_DYNAREC == NEW_DYNAREC_ARM
10416     DebugMessage(M64MSG_VERBOSE, "pre: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d",regmap_pre[i][0],regmap_pre[i][1],regmap_pre[i][2],regmap_pre[i][3],regmap_pre[i][4],regmap_pre[i][5],regmap_pre[i][6],regmap_pre[i][7],regmap_pre[i][8],regmap_pre[i][9],regmap_pre[i][10],regmap_pre[i][12]);
10417     #endif
10418     DebugMessage(M64MSG_VERBOSE, "needs: ");
10419     if(needed_reg[i]&1) DebugMessage(M64MSG_VERBOSE, "eax ");
10420     if((needed_reg[i]>>1)&1) DebugMessage(M64MSG_VERBOSE, "ecx ");
10421     if((needed_reg[i]>>2)&1) DebugMessage(M64MSG_VERBOSE, "edx ");
10422     if((needed_reg[i]>>3)&1) DebugMessage(M64MSG_VERBOSE, "ebx ");
10423     if((needed_reg[i]>>5)&1) DebugMessage(M64MSG_VERBOSE, "ebp ");
10424     if((needed_reg[i]>>6)&1) DebugMessage(M64MSG_VERBOSE, "esi ");
10425     if((needed_reg[i]>>7)&1) DebugMessage(M64MSG_VERBOSE, "edi ");
10426     DebugMessage(M64MSG_VERBOSE, "r:");
10427     for(r=0;r<=CCREG;r++) {
10428       //if(((requires_32bit[i]>>r)&(~unneeded_reg[i]>>r))&1) {
10429       if((requires_32bit[i]>>r)&1) {
10430         if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10431         else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10432         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10433         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10434       }
10435     }
10436     /*DebugMessage(M64MSG_VERBOSE, "pr:");
10437     for(r=0;r<=CCREG;r++) {
10438       //if(((requires_32bit[i]>>r)&(~unneeded_reg[i]>>r))&1) {
10439       if((pr32[i]>>r)&1) {
10440         if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10441         else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10442         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10443         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10444       }
10445     }
10446     if(pr32[i]!=requires_32bit[i]) DebugMessage(M64MSG_ERROR, " OOPS");*/
10447     #if NEW_DYNAREC == NEW_DYNAREC_X86
10448     DebugMessage(M64MSG_VERBOSE, "entry: eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7]);
10449     DebugMessage(M64MSG_VERBOSE, "dirty: ");
10450     if(regs[i].wasdirty&1) DebugMessage(M64MSG_VERBOSE, "eax ");
10451     if((regs[i].wasdirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "ecx ");
10452     if((regs[i].wasdirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "edx ");
10453     if((regs[i].wasdirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "ebx ");
10454     if((regs[i].wasdirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "ebp ");
10455     if((regs[i].wasdirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "esi ");
10456     if((regs[i].wasdirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "edi ");
10457     #endif
10458     #if NEW_DYNAREC == NEW_DYNAREC_ARM
10459     DebugMessage(M64MSG_VERBOSE, "entry: r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d",regs[i].regmap_entry[0],regs[i].regmap_entry[1],regs[i].regmap_entry[2],regs[i].regmap_entry[3],regs[i].regmap_entry[4],regs[i].regmap_entry[5],regs[i].regmap_entry[6],regs[i].regmap_entry[7],regs[i].regmap_entry[8],regs[i].regmap_entry[9],regs[i].regmap_entry[10],regs[i].regmap_entry[12]);
10460     DebugMessage(M64MSG_VERBOSE, "dirty: ");
10461     if(regs[i].wasdirty&1) DebugMessage(M64MSG_VERBOSE, "r0 ");
10462     if((regs[i].wasdirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "r1 ");
10463     if((regs[i].wasdirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "r2 ");
10464     if((regs[i].wasdirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "r3 ");
10465     if((regs[i].wasdirty>>4)&1) DebugMessage(M64MSG_VERBOSE, "r4 ");
10466     if((regs[i].wasdirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "r5 ");
10467     if((regs[i].wasdirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "r6 ");
10468     if((regs[i].wasdirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "r7 ");
10469     if((regs[i].wasdirty>>8)&1) DebugMessage(M64MSG_VERBOSE, "r8 ");
10470     if((regs[i].wasdirty>>9)&1) DebugMessage(M64MSG_VERBOSE, "r9 ");
10471     if((regs[i].wasdirty>>10)&1) DebugMessage(M64MSG_VERBOSE, "r10 ");
10472     if((regs[i].wasdirty>>12)&1) DebugMessage(M64MSG_VERBOSE, "r12 ");
10473     #endif
10474     disassemble_inst(i);
10475     //printf ("ccadj[%d] = %d",i,ccadj[i]);
10476     #if NEW_DYNAREC == NEW_DYNAREC_X86
10477     DebugMessage(M64MSG_VERBOSE, "eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7]);
10478     if(regs[i].dirty&1) DebugMessage(M64MSG_VERBOSE, "eax ");
10479     if((regs[i].dirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "ecx ");
10480     if((regs[i].dirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "edx ");
10481     if((regs[i].dirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "ebx ");
10482     if((regs[i].dirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "ebp ");
10483     if((regs[i].dirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "esi ");
10484     if((regs[i].dirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "edi ");
10485     #endif
10486     #if NEW_DYNAREC == NEW_DYNAREC_ARM
10487     DebugMessage(M64MSG_VERBOSE, "r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",regs[i].regmap[0],regs[i].regmap[1],regs[i].regmap[2],regs[i].regmap[3],regs[i].regmap[4],regs[i].regmap[5],regs[i].regmap[6],regs[i].regmap[7],regs[i].regmap[8],regs[i].regmap[9],regs[i].regmap[10],regs[i].regmap[12]);
10488     if(regs[i].dirty&1) DebugMessage(M64MSG_VERBOSE, "r0 ");
10489     if((regs[i].dirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "r1 ");
10490     if((regs[i].dirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "r2 ");
10491     if((regs[i].dirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "r3 ");
10492     if((regs[i].dirty>>4)&1) DebugMessage(M64MSG_VERBOSE, "r4 ");
10493     if((regs[i].dirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "r5 ");
10494     if((regs[i].dirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "r6 ");
10495     if((regs[i].dirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "r7 ");
10496     if((regs[i].dirty>>8)&1) DebugMessage(M64MSG_VERBOSE, "r8 ");
10497     if((regs[i].dirty>>9)&1) DebugMessage(M64MSG_VERBOSE, "r9 ");
10498     if((regs[i].dirty>>10)&1) DebugMessage(M64MSG_VERBOSE, "r10 ");
10499     if((regs[i].dirty>>12)&1) DebugMessage(M64MSG_VERBOSE, "r12 ");
10500     #endif
10501     if(regs[i].isconst) {
10502       DebugMessage(M64MSG_VERBOSE, "constants: ");
10503       #if NEW_DYNAREC == NEW_DYNAREC_X86
10504       if(regs[i].isconst&1) DebugMessage(M64MSG_VERBOSE, "eax=%x ",(int)constmap[i][0]);
10505       if((regs[i].isconst>>1)&1) DebugMessage(M64MSG_VERBOSE, "ecx=%x ",(int)constmap[i][1]);
10506       if((regs[i].isconst>>2)&1) DebugMessage(M64MSG_VERBOSE, "edx=%x ",(int)constmap[i][2]);
10507       if((regs[i].isconst>>3)&1) DebugMessage(M64MSG_VERBOSE, "ebx=%x ",(int)constmap[i][3]);
10508       if((regs[i].isconst>>5)&1) DebugMessage(M64MSG_VERBOSE, "ebp=%x ",(int)constmap[i][5]);
10509       if((regs[i].isconst>>6)&1) DebugMessage(M64MSG_VERBOSE, "esi=%x ",(int)constmap[i][6]);
10510       if((regs[i].isconst>>7)&1) DebugMessage(M64MSG_VERBOSE, "edi=%x ",(int)constmap[i][7]);
10511       #endif
10512       #if NEW_DYNAREC == NEW_DYNAREC_ARM
10513       if(regs[i].isconst&1) DebugMessage(M64MSG_VERBOSE, "r0=%x ",(int)constmap[i][0]);
10514       if((regs[i].isconst>>1)&1) DebugMessage(M64MSG_VERBOSE, "r1=%x ",(int)constmap[i][1]);
10515       if((regs[i].isconst>>2)&1) DebugMessage(M64MSG_VERBOSE, "r2=%x ",(int)constmap[i][2]);
10516       if((regs[i].isconst>>3)&1) DebugMessage(M64MSG_VERBOSE, "r3=%x ",(int)constmap[i][3]);
10517       if((regs[i].isconst>>4)&1) DebugMessage(M64MSG_VERBOSE, "r4=%x ",(int)constmap[i][4]);
10518       if((regs[i].isconst>>5)&1) DebugMessage(M64MSG_VERBOSE, "r5=%x ",(int)constmap[i][5]);
10519       if((regs[i].isconst>>6)&1) DebugMessage(M64MSG_VERBOSE, "r6=%x ",(int)constmap[i][6]);
10520       if((regs[i].isconst>>7)&1) DebugMessage(M64MSG_VERBOSE, "r7=%x ",(int)constmap[i][7]);
10521       if((regs[i].isconst>>8)&1) DebugMessage(M64MSG_VERBOSE, "r8=%x ",(int)constmap[i][8]);
10522       if((regs[i].isconst>>9)&1) DebugMessage(M64MSG_VERBOSE, "r9=%x ",(int)constmap[i][9]);
10523       if((regs[i].isconst>>10)&1) DebugMessage(M64MSG_VERBOSE, "r10=%x ",(int)constmap[i][10]);
10524       if((regs[i].isconst>>12)&1) DebugMessage(M64MSG_VERBOSE, "r12=%x ",(int)constmap[i][12]);
10525       #endif
10526     }
10527     DebugMessage(M64MSG_VERBOSE, " 32:");
10528     for(r=0;r<=CCREG;r++) {
10529       if((regs[i].is32>>r)&1) {
10530         if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10531         else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10532         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10533         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10534       }
10535     }
10536     /*DebugMessage(M64MSG_VERBOSE, " p32:");
10537     for(r=0;r<=CCREG;r++) {
10538       if((p32[i]>>r)&1) {
10539         if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10540         else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10541         else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10542         else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10543       }
10544     }
10545     if(p32[i]!=regs[i].is32) DebugMessage(M64MSG_VERBOSE, " NO MATCH");*/
10546     if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP) {
10547       #if NEW_DYNAREC == NEW_DYNAREC_X86
10548       DebugMessage(M64MSG_VERBOSE, "branch(%d): eax=%d ecx=%d edx=%d ebx=%d ebp=%d esi=%d edi=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7]);
10549       if(branch_regs[i].dirty&1) DebugMessage(M64MSG_VERBOSE, "eax ");
10550       if((branch_regs[i].dirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "ecx ");
10551       if((branch_regs[i].dirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "edx ");
10552       if((branch_regs[i].dirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "ebx ");
10553       if((branch_regs[i].dirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "ebp ");
10554       if((branch_regs[i].dirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "esi ");
10555       if((branch_regs[i].dirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "edi ");
10556       #endif
10557       #if NEW_DYNAREC == NEW_DYNAREC_ARM
10558       DebugMessage(M64MSG_VERBOSE, "branch(%d): r0=%d r1=%d r2=%d r3=%d r4=%d r5=%d r6=%d r7=%d r8=%d r9=%d r10=%d r12=%d dirty: ",i,branch_regs[i].regmap[0],branch_regs[i].regmap[1],branch_regs[i].regmap[2],branch_regs[i].regmap[3],branch_regs[i].regmap[4],branch_regs[i].regmap[5],branch_regs[i].regmap[6],branch_regs[i].regmap[7],branch_regs[i].regmap[8],branch_regs[i].regmap[9],branch_regs[i].regmap[10],branch_regs[i].regmap[12]);
10559       if(branch_regs[i].dirty&1) DebugMessage(M64MSG_VERBOSE, "r0 ");
10560       if((branch_regs[i].dirty>>1)&1) DebugMessage(M64MSG_VERBOSE, "r1 ");
10561       if((branch_regs[i].dirty>>2)&1) DebugMessage(M64MSG_VERBOSE, "r2 ");
10562       if((branch_regs[i].dirty>>3)&1) DebugMessage(M64MSG_VERBOSE, "r3 ");
10563       if((branch_regs[i].dirty>>4)&1) DebugMessage(M64MSG_VERBOSE, "r4 ");
10564       if((branch_regs[i].dirty>>5)&1) DebugMessage(M64MSG_VERBOSE, "r5 ");
10565       if((branch_regs[i].dirty>>6)&1) DebugMessage(M64MSG_VERBOSE, "r6 ");
10566       if((branch_regs[i].dirty>>7)&1) DebugMessage(M64MSG_VERBOSE, "r7 ");
10567       if((branch_regs[i].dirty>>8)&1) DebugMessage(M64MSG_VERBOSE, "r8 ");
10568       if((branch_regs[i].dirty>>9)&1) DebugMessage(M64MSG_VERBOSE, "r9 ");
10569       if((branch_regs[i].dirty>>10)&1) DebugMessage(M64MSG_VERBOSE, "r10 ");
10570       if((branch_regs[i].dirty>>12)&1) DebugMessage(M64MSG_VERBOSE, "r12 ");
10571       #endif
10572       DebugMessage(M64MSG_VERBOSE, " 32:");
10573       for(r=0;r<=CCREG;r++) {
10574         if((branch_regs[i].is32>>r)&1) {
10575           if(r==CCREG) DebugMessage(M64MSG_VERBOSE, " CC");
10576           else if(r==HIREG) DebugMessage(M64MSG_VERBOSE, " HI");
10577           else if(r==LOREG) DebugMessage(M64MSG_VERBOSE, " LO");
10578           else DebugMessage(M64MSG_VERBOSE, " r%d",r);
10579         }
10580       }
10581     }
10582   }
10583 #endif
10584
10585   /* Pass 8 - Assembly */
10586   linkcount=0;stubcount=0;
10587   ds=0;is_delayslot=0;
10588   cop1_usable=0;
10589   #ifndef DESTRUCTIVE_WRITEBACK
10590   uint64_t is32_pre=0;
10591   u_int dirty_pre=0;
10592   #endif
10593   u_int beginning=(u_int)out;
10594   if((u_int)addr&1) {
10595     ds=1;
10596     pagespan_ds();
10597   }
10598   for(i=0;i<slen;i++)
10599   {
10600     //if(ds) DebugMessage(M64MSG_VERBOSE, "ds: ");
10601 //    if((void*)assem_debug==(void*)printf) disassemble_inst(i);
10602 #if defined( ASSEM_DEBUG )
10603           disassemble_inst(i);
10604 #endif
10605     if(ds) {
10606       ds=0; // Skip delay slot
10607       if(bt[i]) assem_debug("OOPS - branch into delay slot");
10608       instr_addr[i]=0;
10609     } else {
10610       #ifndef DESTRUCTIVE_WRITEBACK
10611       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
10612       {
10613         wb_sx(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,is32_pre,regs[i].was32,
10614               unneeded_reg[i],unneeded_reg_upper[i]);
10615         wb_valid(regmap_pre[i],regs[i].regmap_entry,dirty_pre,regs[i].wasdirty,is32_pre,
10616               unneeded_reg[i],unneeded_reg_upper[i]);
10617       }
10618       is32_pre=regs[i].is32;
10619       dirty_pre=regs[i].dirty;
10620       #endif
10621       // write back
10622       if(i<2||(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000))
10623       {
10624         wb_invalidate(regmap_pre[i],regs[i].regmap_entry,regs[i].wasdirty,regs[i].was32,
10625                       unneeded_reg[i],unneeded_reg_upper[i]);
10626         loop_preload(regmap_pre[i],regs[i].regmap_entry);
10627       }
10628       // branch target entry point
10629       instr_addr[i]=(u_int)out;
10630       assem_debug("<->");
10631       // load regs
10632       if(regs[i].regmap_entry[HOST_CCREG]==CCREG&&regs[i].regmap[HOST_CCREG]!=CCREG)
10633         wb_register(CCREG,regs[i].regmap_entry,regs[i].wasdirty,regs[i].was32);
10634       load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,rs1[i],rs2[i]);
10635       address_generation(i,&regs[i],regs[i].regmap_entry);
10636       load_consts(regmap_pre[i],regs[i].regmap,regs[i].was32,i);
10637       if(itype[i]==RJUMP||itype[i]==UJUMP||itype[i]==CJUMP||itype[i]==SJUMP||itype[i]==FJUMP)
10638       {
10639         // Load the delay slot registers if necessary
10640         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
10641           load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,rs1[i+1],rs1[i+1]);
10642         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
10643           load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,rs2[i+1],rs2[i+1]);
10644         if(itype[i+1]==STORE||itype[i+1]==STORELR||(opcode[i+1]&0x3b)==0x39)
10645           load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,INVCP,INVCP);
10646       }
10647       else if(i+1<slen)
10648       {
10649         // Preload registers for following instruction
10650         if(rs1[i+1]!=rs1[i]&&rs1[i+1]!=rs2[i])
10651           if(rs1[i+1]!=rt1[i]&&rs1[i+1]!=rt2[i])
10652             load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,rs1[i+1],rs1[i+1]);
10653         if(rs2[i+1]!=rs1[i+1]&&rs2[i+1]!=rs1[i]&&rs2[i+1]!=rs2[i])
10654           if(rs2[i+1]!=rt1[i]&&rs2[i+1]!=rt2[i])
10655             load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,rs2[i+1],rs2[i+1]);
10656       }
10657       // TODO: if(is_ooo(i)) address_generation(i+1);
10658       if(itype[i]==CJUMP||itype[i]==FJUMP)
10659         load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,CCREG,CCREG);
10660       if(itype[i]==LOAD||itype[i]==LOADLR||itype[i]==STORE||itype[i]==STORELR||itype[i]==C1LS)
10661         load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,MMREG,ROREG);
10662       if(itype[i]==STORE||itype[i]==STORELR||(opcode[i]&0x3b)==0x39)
10663         load_regs(regs[i].regmap_entry,regs[i].regmap,regs[i].was32,INVCP,INVCP);
10664       if(bt[i]) cop1_usable=0;
10665       // assemble
10666       switch(itype[i]) {
10667         case ALU:
10668           alu_assemble(i,&regs[i]);break;
10669         case IMM16:
10670           imm16_assemble(i,&regs[i]);break;
10671         case SHIFT:
10672           shift_assemble(i,&regs[i]);break;
10673         case SHIFTIMM:
10674           shiftimm_assemble(i,&regs[i]);break;
10675         case LOAD:
10676           load_assemble(i,&regs[i]);break;
10677         case LOADLR:
10678           loadlr_assemble(i,&regs[i]);break;
10679         case STORE:
10680           store_assemble(i,&regs[i]);break;
10681         case STORELR:
10682           storelr_assemble(i,&regs[i]);break;
10683         case COP0:
10684           cop0_assemble(i,&regs[i]);break;
10685         case COP1:
10686           cop1_assemble(i,&regs[i]);break;
10687         case C1LS:
10688           c1ls_assemble(i,&regs[i]);break;
10689         case FCONV:
10690           fconv_assemble(i,&regs[i]);break;
10691         case FLOAT:
10692           float_assemble(i,&regs[i]);break;
10693         case FCOMP:
10694           fcomp_assemble(i,&regs[i]);break;
10695         case MULTDIV:
10696           multdiv_assemble(i,&regs[i]);break;
10697         case MOV:
10698           mov_assemble(i,&regs[i]);break;
10699         case SYSCALL:
10700           syscall_assemble(i,&regs[i]);break;
10701         case UJUMP:
10702           ujump_assemble(i,&regs[i]);ds=1;break;
10703         case RJUMP:
10704           rjump_assemble(i,&regs[i]);ds=1;break;
10705         case CJUMP:
10706           cjump_assemble(i,&regs[i]);ds=1;break;
10707         case SJUMP:
10708           sjump_assemble(i,&regs[i]);ds=1;break;
10709         case FJUMP:
10710           fjump_assemble(i,&regs[i]);ds=1;break;
10711         case SPAN:
10712           pagespan_assemble(i,&regs[i]);break;
10713       }
10714       if(itype[i]==UJUMP||itype[i]==RJUMP||(source[i]>>16)==0x1000)
10715         literal_pool(1024);
10716       else
10717         literal_pool_jumpover(256);
10718     }
10719   }
10720   //assert(itype[i-2]==UJUMP||itype[i-2]==RJUMP||(source[i-2]>>16)==0x1000);
10721   // If the block did not end with an unconditional branch,
10722   // add a jump to the next instruction.
10723   if(i>1) {
10724     if(itype[i-2]!=UJUMP&&itype[i-2]!=RJUMP&&(source[i-2]>>16)!=0x1000&&itype[i-1]!=SPAN) {
10725       assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP&&itype[i-1]!=FJUMP);
10726       assert(i==slen);
10727       if(itype[i-2]!=CJUMP&&itype[i-2]!=SJUMP&&itype[i-2]!=FJUMP) {
10728         store_regs_bt(regs[i-1].regmap,regs[i-1].is32,regs[i-1].dirty,start+i*4);
10729         if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
10730           emit_loadreg(CCREG,HOST_CCREG);
10731         emit_addimm(HOST_CCREG,CLOCK_DIVIDER*(ccadj[i-1]+1),HOST_CCREG);
10732       }
10733       else if(!likely[i-2])
10734       {
10735         store_regs_bt(branch_regs[i-2].regmap,branch_regs[i-2].is32,branch_regs[i-2].dirty,start+i*4);
10736         assert(branch_regs[i-2].regmap[HOST_CCREG]==CCREG);
10737       }
10738       else
10739       {
10740         store_regs_bt(regs[i-2].regmap,regs[i-2].is32,regs[i-2].dirty,start+i*4);
10741         assert(regs[i-2].regmap[HOST_CCREG]==CCREG);
10742       }
10743       add_to_linker((int)out,start+i*4,0);
10744       emit_jmp(0);
10745     }
10746   }
10747   else
10748   {
10749     assert(i>0);
10750     assert(itype[i-1]!=UJUMP&&itype[i-1]!=CJUMP&&itype[i-1]!=SJUMP&&itype[i-1]!=RJUMP&&itype[i-1]!=FJUMP);
10751     store_regs_bt(regs[i-1].regmap,regs[i-1].is32,regs[i-1].dirty,start+i*4);
10752     if(regs[i-1].regmap[HOST_CCREG]!=CCREG)
10753       emit_loadreg(CCREG,HOST_CCREG);
10754     emit_addimm(HOST_CCREG,CLOCK_DIVIDER*(ccadj[i-1]+1),HOST_CCREG);
10755     add_to_linker((int)out,start+i*4,0);
10756     emit_jmp(0);
10757   }
10758
10759   // TODO: delay slot stubs?
10760   // Stubs
10761   for(i=0;i<stubcount;i++)
10762   {
10763     switch(stubs[i][0])
10764     {
10765       case LOADB_STUB:
10766       case LOADH_STUB:
10767       case LOADW_STUB:
10768       case LOADD_STUB:
10769       case LOADBU_STUB:
10770       case LOADHU_STUB:
10771         do_readstub(i);break;
10772       case STOREB_STUB:
10773       case STOREH_STUB:
10774       case STOREW_STUB:
10775       case STORED_STUB:
10776         do_writestub(i);break;
10777       case CC_STUB:
10778         do_ccstub(i);break;
10779       case INVCODE_STUB:
10780         do_invstub(i);break;
10781       case FP_STUB:
10782         do_cop1stub(i);break;
10783       case STORELR_STUB:
10784         do_unalignedwritestub(i);break;
10785     }
10786   }
10787
10788   /* Pass 9 - Linker */
10789   for(i=0;i<linkcount;i++)
10790   {
10791     assem_debug("%8x -> %8x",link_addr[i][0],link_addr[i][1]);
10792     literal_pool(64);
10793     if(!link_addr[i][2])
10794     {
10795       void *stub=out;
10796       void *addr=check_addr(link_addr[i][1]);
10797       emit_extjump(link_addr[i][0],link_addr[i][1]);
10798       if(addr) {
10799         set_jump_target(link_addr[i][0],(int)addr);
10800         add_link(link_addr[i][1],stub);
10801       }
10802       else set_jump_target(link_addr[i][0],(int)stub);
10803     }
10804     else
10805     {
10806       // Internal branch
10807       int target=(link_addr[i][1]-start)>>2;
10808       assert(target>=0&&target<slen);
10809       assert(instr_addr[target]);
10810       //#ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
10811       //set_jump_target_fillslot(link_addr[i][0],instr_addr[target],link_addr[i][2]>>1);
10812       //#else
10813       set_jump_target(link_addr[i][0],instr_addr[target]);
10814       //#endif
10815     }
10816   }
10817   // External Branch Targets (jump_in)
10818   if(copy+slen*4>(void *)shadow+sizeof(shadow)) copy=shadow;
10819   for(i=0;i<slen;i++)
10820   {
10821     if(bt[i]||i==0)
10822     {
10823       if(instr_addr[i]) // TODO - delay slots (=null)
10824       {
10825         u_int vaddr=start+i*4;
10826         u_int page=(0x80000000^vaddr)>>12;
10827         u_int vpage=page;
10828         if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[page^0x80000]^0x80000000)>>12;
10829         if(page>2048) page=2048+(page&2047);
10830         if(vpage>262143&&tlb_LUT_r[vaddr>>12]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
10831         if(vpage>2048) vpage=2048+(vpage&2047);
10832         literal_pool(256);
10833         //if(!(is32[i]&(~unneeded_reg_upper[i])&~(1LL<<CCREG)))
10834         if(!requires_32bit[i])
10835         {
10836           assem_debug("%8x (%d) <- %8x",instr_addr[i],i,start+i*4);
10837           assem_debug("jump_in: %x",start+i*4);
10838           ll_add(jump_dirty+vpage,vaddr,(void *)out);
10839           int entry_point=do_dirty_stub(i);
10840           ll_add(jump_in+page,vaddr,(void *)entry_point);
10841           // If there was an existing entry in the hash table,
10842           // replace it with the new address.
10843           // Don't add new entries.  We'll insert the
10844           // ones that actually get used in check_addr().
10845           u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
10846           if(ht_bin[0]==vaddr) {
10847             ht_bin[1]=entry_point;
10848           }
10849           if(ht_bin[2]==vaddr) {
10850             ht_bin[3]=entry_point;
10851           }
10852         }
10853         else
10854         {
10855           u_int r=requires_32bit[i]|!!(requires_32bit[i]>>32);
10856           assem_debug("%8x (%d) <- %8x",instr_addr[i],i,start+i*4);
10857           assem_debug("jump_in: %x (restricted - %x)",start+i*4,r);
10858           //int entry_point=(int)out;
10859           ////assem_debug("entry_point: %x",entry_point);
10860           //load_regs_entry(i);
10861           //if(entry_point==(int)out)
10862           //  entry_point=instr_addr[i];
10863           //else
10864           //  emit_jmp(instr_addr[i]);
10865           //ll_add_32(jump_in+page,vaddr,r,(void *)entry_point);
10866           ll_add_32(jump_dirty+vpage,vaddr,r,(void *)out);
10867           int entry_point=do_dirty_stub(i);
10868           ll_add_32(jump_in+page,vaddr,r,(void *)entry_point);
10869         }
10870       }
10871     }
10872   }
10873   // Write out the literal pool if necessary
10874   literal_pool(0);
10875   #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
10876   // Align code
10877   if(((u_int)out)&7) emit_addnop(13);
10878   #endif
10879   assert((u_int)out-beginning<MAX_OUTPUT_BLOCK_SIZE);
10880   //DebugMessage(M64MSG_VERBOSE, "shadow buffer: %x-%x",(int)copy,(int)copy+slen*4);
10881   memcpy(copy,source,slen*4);
10882   copy+=slen*4;
10883
10884   #if NEW_DYNAREC == NEW_DYNAREC_ARM
10885   __clear_cache((void *)beginning,out);
10886   //cacheflush((void *)beginning,out,0);
10887   #endif
10888
10889   // If we're within 256K of the end of the buffer,
10890   // start over from the beginning. (Is 256K enough?)
10891   if(out > (u_char *)(base_addr+(1<<TARGET_SIZE_2)-MAX_OUTPUT_BLOCK_SIZE-JUMP_TABLE_SIZE))
10892     out=(u_char *)base_addr;
10893
10894   // Trap writes to any of the pages we compiled
10895   for(i=start>>12;i<=(start+slen*4)>>12;i++) {
10896     invalid_code[i]=0;
10897     memory_map[i]|=0x40000000;
10898     if((signed int)start>=(signed int)0xC0000000) {
10899       assert(using_tlb);
10900       j=(((u_int)i<<12)+(memory_map[i]<<2)-(u_int)rdram+(u_int)0x80000000)>>12;
10901       invalid_code[j]=0;
10902       memory_map[j]|=0x40000000;
10903       //DebugMessage(M64MSG_VERBOSE, "write protect physical page: %x (virtual %x)",j<<12,start);
10904     }
10905   }
10906
10907   /* Pass 10 - Free memory by expiring oldest blocks */
10908
10909   int end=((((intptr_t)out-(intptr_t)base_addr)>>(TARGET_SIZE_2-16))+16384)&65535;
10910   while(expirep!=end)
10911   {
10912     int shift=TARGET_SIZE_2-3; // Divide into 8 blocks
10913     int base=(int)base_addr+((expirep>>13)<<shift); // Base address of this block
10914     inv_debug("EXP: Phase %d\n",expirep);
10915     switch((expirep>>11)&3)
10916     {
10917       case 0:
10918         // Clear jump_in and jump_dirty
10919         ll_remove_matching_addrs(jump_in+(expirep&2047),base,shift);
10920         ll_remove_matching_addrs(jump_dirty+(expirep&2047),base,shift);
10921         ll_remove_matching_addrs(jump_in+2048+(expirep&2047),base,shift);
10922         ll_remove_matching_addrs(jump_dirty+2048+(expirep&2047),base,shift);
10923         break;
10924       case 1:
10925         // Clear pointers
10926         ll_kill_pointers(jump_out[expirep&2047],base,shift);
10927         ll_kill_pointers(jump_out[(expirep&2047)+2048],base,shift);
10928         break;
10929       case 2:
10930         // Clear hash table
10931         for(i=0;i<32;i++) {
10932           u_int *ht_bin=hash_table[((expirep&2047)<<5)+i];
10933           if((ht_bin[3]>>shift)==(base>>shift) ||
10934              ((ht_bin[3]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
10935             inv_debug("EXP: Remove hash %x -> %x\n",ht_bin[2],ht_bin[3]);
10936             ht_bin[2]=ht_bin[3]=-1;
10937           }
10938           if((ht_bin[1]>>shift)==(base>>shift) ||
10939              ((ht_bin[1]-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(base>>shift)) {
10940             inv_debug("EXP: Remove hash %x -> %x\n",ht_bin[0],ht_bin[1]);
10941             ht_bin[0]=ht_bin[2];
10942             ht_bin[1]=ht_bin[3];
10943             ht_bin[2]=ht_bin[3]=-1;
10944           }
10945         }
10946         break;
10947       case 3:
10948         // Clear jump_out
10949         #if NEW_DYNAREC == NEW_DYNAREC_ARM
10950         if((expirep&2047)==0)
10951           do_clear_cache();
10952         #endif
10953         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
10954         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
10955         break;
10956     }
10957     expirep=(expirep+1)&65535;
10958   }
10959   return 0;
10960 }
10961
10962 void TLBWI_new(void)
10963 {
10964   unsigned int i;
10965   /* Remove old entries */
10966   unsigned int old_start_even=tlb_e[Index&0x3F].start_even;
10967   unsigned int old_end_even=tlb_e[Index&0x3F].end_even;
10968   unsigned int old_start_odd=tlb_e[Index&0x3F].start_odd;
10969   unsigned int old_end_odd=tlb_e[Index&0x3F].end_odd;
10970   for (i=old_start_even>>12; i<=old_end_even>>12; i++)
10971   {
10972     if(i<0x80000||i>0xBFFFF)
10973     {
10974       invalidate_block(i);
10975       memory_map[i]=-1;
10976     }
10977   }
10978   for (i=old_start_odd>>12; i<=old_end_odd>>12; i++)
10979   {
10980     if(i<0x80000||i>0xBFFFF)
10981     {
10982       invalidate_block(i);
10983       memory_map[i]=-1;
10984     }
10985   }
10986   cached_interpreter_table.TLBWI();
10987   //DebugMessage(M64MSG_VERBOSE, "TLBWI: index=%d",Index);
10988   //DebugMessage(M64MSG_VERBOSE, "TLBWI: start_even=%x end_even=%x phys_even=%x v=%d d=%d",tlb_e[Index&0x3F].start_even,tlb_e[Index&0x3F].end_even,tlb_e[Index&0x3F].phys_even,tlb_e[Index&0x3F].v_even,tlb_e[Index&0x3F].d_even);
10989   //DebugMessage(M64MSG_VERBOSE, "TLBWI: start_odd=%x end_odd=%x phys_odd=%x v=%d d=%d",tlb_e[Index&0x3F].start_odd,tlb_e[Index&0x3F].end_odd,tlb_e[Index&0x3F].phys_odd,tlb_e[Index&0x3F].v_odd,tlb_e[Index&0x3F].d_odd);
10990   /* Combine tlb_LUT_r, tlb_LUT_w, and invalid_code into a single table
10991      for fast look up. */
10992   for (i=tlb_e[Index&0x3F].start_even>>12; i<=tlb_e[Index&0x3F].end_even>>12; i++)
10993   {
10994     //DebugMessage(M64MSG_VERBOSE, "%x: r:%8x w:%8x",i,tlb_LUT_r[i],tlb_LUT_w[i]);
10995     if(i<0x80000||i>0xBFFFF)
10996     {
10997       if(tlb_LUT_r[i]) {
10998         memory_map[i]=((tlb_LUT_r[i]&0xFFFFF000)-(i<<12)+(unsigned int)rdram-0x80000000)>>2;
10999         // FIXME: should make sure the physical page is invalid too
11000         if(!tlb_LUT_w[i]||!invalid_code[i]) {
11001           memory_map[i]|=0x40000000; // Write protect
11002         }else{
11003           assert(tlb_LUT_r[i]==tlb_LUT_w[i]);
11004         }
11005         if(!using_tlb) DebugMessage(M64MSG_VERBOSE, "Enabled TLB");
11006         // Tell the dynamic recompiler to generate tlb lookup code
11007         using_tlb=1;
11008       }
11009       else memory_map[i]=-1;
11010     }
11011     //DebugMessage(M64MSG_VERBOSE, "memory_map[%x]: %8x (+%8x)",i,memory_map[i],memory_map[i]<<2);
11012   }
11013   for (i=tlb_e[Index&0x3F].start_odd>>12; i<=tlb_e[Index&0x3F].end_odd>>12; i++)
11014   {
11015     //DebugMessage(M64MSG_VERBOSE, "%x: r:%8x w:%8x",i,tlb_LUT_r[i],tlb_LUT_w[i]);
11016     if(i<0x80000||i>0xBFFFF)
11017     {
11018       if(tlb_LUT_r[i]) {
11019         memory_map[i]=((tlb_LUT_r[i]&0xFFFFF000)-(i<<12)+(unsigned int)rdram-0x80000000)>>2;
11020         // FIXME: should make sure the physical page is invalid too
11021         if(!tlb_LUT_w[i]||!invalid_code[i]) {
11022           memory_map[i]|=0x40000000; // Write protect
11023         }else{
11024           assert(tlb_LUT_r[i]==tlb_LUT_w[i]);
11025         }
11026         if(!using_tlb) DebugMessage(M64MSG_VERBOSE, "Enabled TLB");
11027         // Tell the dynamic recompiler to generate tlb lookup code
11028         using_tlb=1;
11029       }
11030       else memory_map[i]=-1;
11031     }
11032     //DebugMessage(M64MSG_VERBOSE, "memory_map[%x]: %8x (+%8x)",i,memory_map[i],memory_map[i]<<2);
11033   }
11034 }
11035
11036 void TLBWR_new(void)
11037 {
11038   unsigned int i;
11039   Random = (Count/2 % (32 - Wired)) + Wired;
11040   /* Remove old entries */
11041   unsigned int old_start_even=tlb_e[Random&0x3F].start_even;
11042   unsigned int old_end_even=tlb_e[Random&0x3F].end_even;
11043   unsigned int old_start_odd=tlb_e[Random&0x3F].start_odd;
11044   unsigned int old_end_odd=tlb_e[Random&0x3F].end_odd;
11045   for (i=old_start_even>>12; i<=old_end_even>>12; i++)
11046   {
11047     if(i<0x80000||i>0xBFFFF)
11048     {
11049       invalidate_block(i);
11050       memory_map[i]=-1;
11051     }
11052   }
11053   for (i=old_start_odd>>12; i<=old_end_odd>>12; i++)
11054   {
11055     if(i<0x80000||i>0xBFFFF)
11056     {
11057       invalidate_block(i);
11058       memory_map[i]=-1;
11059     }
11060   }
11061   cached_interpreter_table.TLBWR();
11062   /* Combine tlb_LUT_r, tlb_LUT_w, and invalid_code into a single table
11063      for fast look up. */
11064   for (i=tlb_e[Random&0x3F].start_even>>12; i<=tlb_e[Random&0x3F].end_even>>12; i++)
11065   {
11066     //DebugMessage(M64MSG_VERBOSE, "%x: r:%8x w:%8x",i,tlb_LUT_r[i],tlb_LUT_w[i]);
11067     if(i<0x80000||i>0xBFFFF)
11068     {
11069       if(tlb_LUT_r[i]) {
11070         memory_map[i]=((tlb_LUT_r[i]&0xFFFFF000)-(i<<12)+(unsigned int)rdram-0x80000000)>>2;
11071         // FIXME: should make sure the physical page is invalid too
11072         if(!tlb_LUT_w[i]||!invalid_code[i]) {
11073           memory_map[i]|=0x40000000; // Write protect
11074         }else{
11075           assert(tlb_LUT_r[i]==tlb_LUT_w[i]);
11076         }
11077         if(!using_tlb) DebugMessage(M64MSG_VERBOSE, "Enabled TLB");
11078         // Tell the dynamic recompiler to generate tlb lookup code
11079         using_tlb=1;
11080       }
11081       else memory_map[i]=-1;
11082     }
11083     //DebugMessage(M64MSG_VERBOSE, "memory_map[%x]: %8x (+%8x)",i,memory_map[i],memory_map[i]<<2);
11084   }
11085   for (i=tlb_e[Random&0x3F].start_odd>>12; i<=tlb_e[Random&0x3F].end_odd>>12; i++)
11086   {
11087     //DebugMessage(M64MSG_VERBOSE, "%x: r:%8x w:%8x",i,tlb_LUT_r[i],tlb_LUT_w[i]);
11088     if(i<0x80000||i>0xBFFFF)
11089     {
11090       if(tlb_LUT_r[i]) {
11091         memory_map[i]=((tlb_LUT_r[i]&0xFFFFF000)-(i<<12)+(unsigned int)rdram-0x80000000)>>2;
11092         // FIXME: should make sure the physical page is invalid too
11093         if(!tlb_LUT_w[i]||!invalid_code[i]) {
11094           memory_map[i]|=0x40000000; // Write protect
11095         }else{
11096           assert(tlb_LUT_r[i]==tlb_LUT_w[i]);
11097         }
11098         if(!using_tlb) DebugMessage(M64MSG_VERBOSE, "Enabled TLB");
11099         // Tell the dynamic recompiler to generate tlb lookup code
11100         using_tlb=1;
11101       }
11102       else memory_map[i]=-1;
11103     }
11104     //DebugMessage(M64MSG_VERBOSE, "memory_map[%x]: %8x (+%8x)",i,memory_map[i],memory_map[i]<<2);
11105   }
11106 }