libpcsxcore/new_dynarec/new_dynarec.c

   1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
   2  *   Mupen64plus - new_dynarec.c                                           *
   3  *   Copyright (C) 2009-2010 Ari64                                         *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.          *
  19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  20
  21 #include <stdlib.h>
  22 #include <stdint.h> //include for uint64_t
  23 #include <assert.h>
  24
  25 #include "emu_if.h" //emulator interface
  26
  27 #include <sys/mman.h>
  28
  29 #ifdef __i386__
  30 #include "assem_x86.h"
  31 #endif
  32 #ifdef __x86_64__
  33 #include "assem_x64.h"
  34 #endif
  35 #ifdef __arm__
  36 #include "assem_arm.h"
  37 #endif
  38
  39 #define MAXBLOCK 4096
  40 #define MAX_OUTPUT_BLOCK_SIZE 262144
  41 #define CLOCK_DIVIDER 2
  42
  43 struct regstat
  44 {
  45   signed char regmap_entry[HOST_REGS];
  46   signed char regmap[HOST_REGS];
  47   uint64_t was32;
  48   uint64_t is32;
  49   uint64_t wasdirty;
  50   uint64_t dirty;
  51   uint64_t u;
  52   uint64_t uu;
  53   u_int wasconst;
  54   u_int isconst;
  55   uint64_t constmap[HOST_REGS];
  56 };
  57
  58 struct ll_entry
  59 {
  60   u_int vaddr;
  61   u_int reg32;
  62   void *addr;
  63   struct ll_entry *next;
  64 };
  65
  66   u_int start;
  67   u_int *source;
  68   u_int pagelimit;
  69   char insn[MAXBLOCK][10];
  70   u_char itype[MAXBLOCK];
  71   u_char opcode[MAXBLOCK];
  72   u_char opcode2[MAXBLOCK];
  73   u_char bt[MAXBLOCK];
  74   u_char rs1[MAXBLOCK];
  75   u_char rs2[MAXBLOCK];
  76   u_char rt1[MAXBLOCK];
  77   u_char rt2[MAXBLOCK];
  78   u_char us1[MAXBLOCK];
  79   u_char us2[MAXBLOCK];
  80   u_char dep1[MAXBLOCK];
  81   u_char dep2[MAXBLOCK];
  82   u_char lt1[MAXBLOCK];
  83   int imm[MAXBLOCK];
  84   u_int ba[MAXBLOCK];
  85   char likely[MAXBLOCK];
  86   char is_ds[MAXBLOCK];
  87   uint64_t unneeded_reg[MAXBLOCK];
  88   uint64_t unneeded_reg_upper[MAXBLOCK];
  89   uint64_t branch_unneeded_reg[MAXBLOCK];
  90   uint64_t branch_unneeded_reg_upper[MAXBLOCK];
  91   uint64_t p32[MAXBLOCK];
  92   uint64_t pr32[MAXBLOCK];
  93   signed char regmap_pre[MAXBLOCK][HOST_REGS];
  94   signed char regmap[MAXBLOCK][HOST_REGS];
  95   signed char regmap_entry[MAXBLOCK][HOST_REGS];
  96   uint64_t constmap[MAXBLOCK][HOST_REGS];
  97   uint64_t known_value[HOST_REGS];
  98   u_int known_reg;
  99   struct regstat regs[MAXBLOCK];
 100   struct regstat branch_regs[MAXBLOCK];
 101   u_int needed_reg[MAXBLOCK];
 102   uint64_t requires_32bit[MAXBLOCK];
 103   u_int wont_dirty[MAXBLOCK];
 104   u_int will_dirty[MAXBLOCK];
 105   int ccadj[MAXBLOCK];
 106   int slen;
 107   u_int instr_addr[MAXBLOCK];
 108   u_int link_addr[MAXBLOCK][3];
 109   int linkcount;
 110   u_int stubs[MAXBLOCK*3][8];
 111   int stubcount;
 112   u_int literals[1024][2];
 113   int literalcount;
 114   int is_delayslot;
 115   int cop1_usable;
 116   u_char *out;
 117   struct ll_entry *jump_in[4096];
 118   struct ll_entry *jump_out[4096];
 119   struct ll_entry *jump_dirty[4096];
 120   u_int hash_table[65536][4]  __attribute__((aligned(16)));
 121   char shadow[1048576]  __attribute__((aligned(16)));
 122   void *copy;
 123   int expirep;
 124   u_int using_tlb;
 125   u_int stop_after_jal;
 126   extern u_char restore_candidate[512];
 127   extern int cycle_count;
 128
 129   /* registers that may be allocated */
 130   /* 1-31 gpr */
 131 #define HIREG 32 // hi
 132 #define LOREG 33 // lo
 133 #define FSREG 34 // FPU status (FCSR)
 134 #define CSREG 35 // Coprocessor status
 135 #define CCREG 36 // Cycle count
 136 #define INVCP 37 // Pointer to invalid_code
 137 #define TEMPREG 38
 138 #define FTEMP 38 // FPU/LDL/LDR temporary register
 139 #define PTEMP 39 // Prefetch temporary register
 140 #define TLREG 40 // TLB mapping offset
 141 #define RHASH 41 // Return address hash
 142 #define RHTBL 42 // Return address hash table address
 143 #define RTEMP 43 // JR/JALR address register
 144 #define MAXREG 43
 145 #define AGEN1 44 // Address generation temporary register
 146 #define AGEN2 45 // Address generation temporary register
 147 #define MGEN1 46 // Maptable address generation temporary register
 148 #define MGEN2 47 // Maptable address generation temporary register
 149 #define BTREG 48 // Branch target temporary register
 150
 151   /* instruction types */
 152 #define NOP 0     // No operation
 153 #define LOAD 1    // Load
 154 #define STORE 2   // Store
 155 #define LOADLR 3  // Unaligned load
 156 #define STORELR 4 // Unaligned store
 157 #define MOV 5     // Move
 158 #define ALU 6     // Arithmetic/logic
 159 #define MULTDIV 7 // Multiply/divide
 160 #define SHIFT 8   // Shift by register
 161 #define SHIFTIMM 9// Shift by immediate
 162 #define IMM16 10  // 16-bit immediate
 163 #define RJUMP 11  // Unconditional jump to register
 164 #define UJUMP 12  // Unconditional jump
 165 #define CJUMP 13  // Conditional branch (BEQ/BNE/BGTZ/BLEZ)
 166 #define SJUMP 14  // Conditional branch (regimm format)
 167 #define COP0 15   // Coprocessor 0
 168 #define COP1 16   // Coprocessor 1
 169 #define C1LS 17   // Coprocessor 1 load/store
 170 #define FJUMP 18  // Conditional branch (floating point)
 171 #define FLOAT 19  // Floating point unit
 172 #define FCONV 20  // Convert integer to float
 173 #define FCOMP 21  // Floating point compare (sets FSREG)
 174 #define SYSCALL 22// SYSCALL
 175 #define OTHER 23  // Other
 176 #define SPAN 24   // Branch/delay slot spans 2 pages
 177 #define NI 25     // Not implemented
 178 #define HLECALL 26// PCSX fake opcodes for HLE
 179 #define COP2 27   // Coprocessor 2 move
 180 #define C2LS 28   // Coprocessor 2 load/store
 181 #define C2OP 29   // Coprocessor 2 operation
 182 #define INTCALL 30// Call interpreter to handle rare corner cases
 183
 184   /* stubs */
 185 #define CC_STUB 1
 186 #define FP_STUB 2
 187 #define LOADB_STUB 3
 188 #define LOADH_STUB 4
 189 #define LOADW_STUB 5
 190 #define LOADD_STUB 6
 191 #define LOADBU_STUB 7
 192 #define LOADHU_STUB 8
 193 #define STOREB_STUB 9
 194 #define STOREH_STUB 10
 195 #define STOREW_STUB 11
 196 #define STORED_STUB 12
 197 #define STORELR_STUB 13
 198 #define INVCODE_STUB 14
 199
 200   /* branch codes */
 201 #define TAKEN 1
 202 #define NOTTAKEN 2
 203 #define NULLDS 3
 204
 205 // asm linkage
 206 int new_recompile_block(int addr);
 207 void *get_addr_ht(u_int vaddr);
 208 void invalidate_block(u_int block);
 209 void invalidate_addr(u_int addr);
 210 void remove_hash(int vaddr);
 211 void jump_vaddr();
 212 void dyna_linker();
 213 void dyna_linker_ds();
 214 void verify_code();
 215 void verify_code_vm();
 216 void verify_code_ds();
 217 void cc_interrupt();
 218 void fp_exception();
 219 void fp_exception_ds();
 220 void jump_syscall();
 221 void jump_syscall_hle();
 222 void jump_eret();
 223 void jump_hlecall();
 224 void jump_intcall();
 225 void new_dyna_leave();
 226
 227 // TLB
 228 void TLBWI_new();
 229 void TLBWR_new();
 230 void read_nomem_new();
 231 void read_nomemb_new();
 232 void read_nomemh_new();
 233 void read_nomemd_new();
 234 void write_nomem_new();
 235 void write_nomemb_new();
 236 void write_nomemh_new();
 237 void write_nomemd_new();
 238 void write_rdram_new();
 239 void write_rdramb_new();
 240 void write_rdramh_new();
 241 void write_rdramd_new();
 242 extern u_int memory_map[1048576];
 243
 244 // Needed by assembler
 245 void wb_register(signed char r,signed char regmap[],uint64_t dirty,uint64_t is32);
 246 void wb_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty);
 247 void wb_needed_dirtys(signed char i_regmap[],uint64_t i_is32,uint64_t i_dirty,int addr);
 248 void load_all_regs(signed char i_regmap[]);
 249 void load_needed_regs(signed char i_regmap[],signed char next_regmap[]);
 250 void load_regs_entry(int t);
 251 void load_all_consts(signed char regmap[],int is32,u_int dirty,int i);
 252
 253 int tracedebug=0;
 254
 255 //#define DEBUG_CYCLE_COUNT 1
 256
 257 void nullf() {}
 258 //#define assem_debug printf
 259 //#define inv_debug printf
 260 #define assem_debug nullf
 261 #define inv_debug nullf
 262
 263 static void tlb_hacks()
 264 {
 265 #ifndef DISABLE_TLB
 266   // Goldeneye hack
 267   if (strncmp((char *) ROM_HEADER->nom, "GOLDENEYE",9) == 0)
 268   {
 269     u_int addr;
 270     int n;
 271     switch (ROM_HEADER->Country_code&0xFF)
 272     {
 273       case 0x45: // U
 274         addr=0x34b30;
 275         break;
 276       case 0x4A: // J
 277         addr=0x34b70;
 278         break;
 279       case 0x50: // E
 280         addr=0x329f0;
 281         break;
 282       default:
 283         // Unknown country code
 284         addr=0;
 285         break;
 286     }
 287     u_int rom_addr=(u_int)rom;
 288     #ifdef ROM_COPY
 289     // Since memory_map is 32-bit, on 64-bit systems the rom needs to be
 290     // in the lower 4G of memory to use this hack.  Copy it if necessary.
 291     if((void *)rom>(void *)0xffffffff) {
 292       munmap(ROM_COPY, 67108864);
 293       if(mmap(ROM_COPY, 12582912,
 294               PROT_READ | PROT_WRITE,
 295               MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
 296               -1, 0) <= 0) {printf("mmap() failed\n");}
 297       memcpy(ROM_COPY,rom,12582912);
 298       rom_addr=(u_int)ROM_COPY;
 299     }
 300     #endif
 301     if(addr) {
 302       for(n=0x7F000;n<0x80000;n++) {
 303         memory_map[n]=(((u_int)(rom_addr+addr-0x7F000000))>>2)|0x40000000;
 304       }
 305     }
 306   }
 307 #endif
 308 }
 309
 310 static u_int get_page(u_int vaddr)
 311 {
 312   u_int page=(vaddr^0x80000000)>>12;
 313 #ifndef DISABLE_TLB
 314   if(page>262143&&tlb_LUT_r[vaddr>>12]) page=(tlb_LUT_r[vaddr>>12]^0x80000000)>>12;
 315 #endif
 316   if(page>2048) page=2048+(page&2047);
 317   return page;
 318 }
 319
 320 static u_int get_vpage(u_int vaddr)
 321 {
 322   u_int vpage=(vaddr^0x80000000)>>12;
 323 #ifndef DISABLE_TLB
 324   if(vpage>262143&&tlb_LUT_r[vaddr>>12]) vpage&=2047; // jump_dirty uses a hash of the virtual address instead
 325 #endif
 326   if(vpage>2048) vpage=2048+(vpage&2047);
 327   return vpage;
 328 }
 329
 330 // Get address from virtual address
 331 // This is called from the recompiled JR/JALR instructions
 332 void *get_addr(u_int vaddr)
 333 {
 334   u_int page=get_page(vaddr);
 335   u_int vpage=get_vpage(vaddr);
 336   struct ll_entry *head;
 337   //printf("TRACE: count=%d next=%d (get_addr %x,page %d)\n",Count,next_interupt,vaddr,page);
 338   head=jump_in[page];
 339   while(head!=NULL) {
 340     if(head->vaddr==vaddr&&head->reg32==0) {
 341   //printf("TRACE: count=%d next=%d (get_addr match %x: %x)\n",Count,next_interupt,vaddr,(int)head->addr);
 342       int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 343       ht_bin[3]=ht_bin[1];
 344       ht_bin[2]=ht_bin[0];
 345       ht_bin[1]=(int)head->addr;
 346       ht_bin[0]=vaddr;
 347       return head->addr;
 348     }
 349     head=head->next;
 350   }
 351   head=jump_dirty[vpage];
 352   while(head!=NULL) {
 353     if(head->vaddr==vaddr&&head->reg32==0) {
 354       //printf("TRACE: count=%d next=%d (get_addr match dirty %x: %x)\n",Count,next_interupt,vaddr,(int)head->addr);
 355       // Don't restore blocks which are about to expire from the cache
 356       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 357       if(verify_dirty(head->addr)) {
 358         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 359         invalid_code[vaddr>>12]=0;
 360         memory_map[vaddr>>12]|=0x40000000;
 361         if(vpage<2048) {
 362 #ifndef DISABLE_TLB
 363           if(tlb_LUT_r[vaddr>>12]) {
 364             invalid_code[tlb_LUT_r[vaddr>>12]>>12]=0;
 365             memory_map[tlb_LUT_r[vaddr>>12]>>12]|=0x40000000;
 366           }
 367 #endif
 368           restore_candidate[vpage>>3]|=1<<(vpage&7);
 369         }
 370         else restore_candidate[page>>3]|=1<<(page&7);
 371         int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 372         if(ht_bin[0]==vaddr) {
 373           ht_bin[1]=(int)head->addr; // Replace existing entry
 374         }
 375         else
 376         {
 377           ht_bin[3]=ht_bin[1];
 378           ht_bin[2]=ht_bin[0];
 379           ht_bin[1]=(int)head->addr;
 380           ht_bin[0]=vaddr;
 381         }
 382         return head->addr;
 383       }
 384     }
 385     head=head->next;
 386   }
 387   //printf("TRACE: count=%d next=%d (get_addr no-match %x)\n",Count,next_interupt,vaddr);
 388   int r=new_recompile_block(vaddr);
 389   if(r==0) return get_addr(vaddr);
 390   // Execute in unmapped page, generate pagefault execption
 391   Status|=2;
 392   Cause=(vaddr<<31)|0x8;
 393   EPC=(vaddr&1)?vaddr-5:vaddr;
 394   BadVAddr=(vaddr&~1);
 395   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 396   EntryHi=BadVAddr&0xFFFFE000;
 397   return get_addr_ht(0x80000000);
 398 }
 399 // Look up address in hash table first
 400 void *get_addr_ht(u_int vaddr)
 401 {
 402   //printf("TRACE: count=%d next=%d (get_addr_ht %x)\n",Count,next_interupt,vaddr);
 403   int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 404   if(ht_bin[0]==vaddr) return (void *)ht_bin[1];
 405   if(ht_bin[2]==vaddr) return (void *)ht_bin[3];
 406   return get_addr(vaddr);
 407 }
 408
 409 void *get_addr_32(u_int vaddr,u_int flags)
 410 {
 411 #ifdef FORCE32
 412   return get_addr(vaddr);
 413 #else
 414   //printf("TRACE: count=%d next=%d (get_addr_32 %x,flags %x)\n",Count,next_interupt,vaddr,flags);
 415   int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 416   if(ht_bin[0]==vaddr) return (void *)ht_bin[1];
 417   if(ht_bin[2]==vaddr) return (void *)ht_bin[3];
 418   u_int page=get_page(vaddr);
 419   u_int vpage=get_vpage(vaddr);
 420   struct ll_entry *head;
 421   head=jump_in[page];
 422   while(head!=NULL) {
 423     if(head->vaddr==vaddr&&(head->reg32&flags)==0) {
 424       //printf("TRACE: count=%d next=%d (get_addr_32 match %x: %x)\n",Count,next_interupt,vaddr,(int)head->addr);
 425       if(head->reg32==0) {
 426         int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 427         if(ht_bin[0]==-1) {
 428           ht_bin[1]=(int)head->addr;
 429           ht_bin[0]=vaddr;
 430         }else if(ht_bin[2]==-1) {
 431           ht_bin[3]=(int)head->addr;
 432           ht_bin[2]=vaddr;
 433         }
 434         //ht_bin[3]=ht_bin[1];
 435         //ht_bin[2]=ht_bin[0];
 436         //ht_bin[1]=(int)head->addr;
 437         //ht_bin[0]=vaddr;
 438       }
 439       return head->addr;
 440     }
 441     head=head->next;
 442   }
 443   head=jump_dirty[vpage];
 444   while(head!=NULL) {
 445     if(head->vaddr==vaddr&&(head->reg32&flags)==0) {
 446       //printf("TRACE: count=%d next=%d (get_addr_32 match dirty %x: %x)\n",Count,next_interupt,vaddr,(int)head->addr);
 447       // Don't restore blocks which are about to expire from the cache
 448       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 449       if(verify_dirty(head->addr)) {
 450         //printf("restore candidate: %x (%d) d=%d\n",vaddr,page,invalid_code[vaddr>>12]);
 451         invalid_code[vaddr>>12]=0;
 452         memory_map[vaddr>>12]|=0x40000000;
 453         if(vpage<2048) {
 454 #ifndef DISABLE_TLB
 455           if(tlb_LUT_r[vaddr>>12]) {
 456             invalid_code[tlb_LUT_r[vaddr>>12]>>12]=0;
 457             memory_map[tlb_LUT_r[vaddr>>12]>>12]|=0x40000000;
 458           }
 459 #endif
 460           restore_candidate[vpage>>3]|=1<<(vpage&7);
 461         }
 462         else restore_candidate[page>>3]|=1<<(page&7);
 463         if(head->reg32==0) {
 464           int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 465           if(ht_bin[0]==-1) {
 466             ht_bin[1]=(int)head->addr;
 467             ht_bin[0]=vaddr;
 468           }else if(ht_bin[2]==-1) {
 469             ht_bin[3]=(int)head->addr;
 470             ht_bin[2]=vaddr;
 471           }
 472           //ht_bin[3]=ht_bin[1];
 473           //ht_bin[2]=ht_bin[0];
 474           //ht_bin[1]=(int)head->addr;
 475           //ht_bin[0]=vaddr;
 476         }
 477         return head->addr;
 478       }
 479     }
 480     head=head->next;
 481   }
 482   //printf("TRACE: count=%d next=%d (get_addr_32 no-match %x,flags %x)\n",Count,next_interupt,vaddr,flags);
 483   int r=new_recompile_block(vaddr);
 484   if(r==0) return get_addr(vaddr);
 485   // Execute in unmapped page, generate pagefault execption
 486   Status|=2;
 487   Cause=(vaddr<<31)|0x8;
 488   EPC=(vaddr&1)?vaddr-5:vaddr;
 489   BadVAddr=(vaddr&~1);
 490   Context=(Context&0xFF80000F)|((BadVAddr>>9)&0x007FFFF0);
 491   EntryHi=BadVAddr&0xFFFFE000;
 492   return get_addr_ht(0x80000000);
 493 #endif
 494 }
 495
 496 void clear_all_regs(signed char regmap[])
 497 {
 498   int hr;
 499   for (hr=0;hr<HOST_REGS;hr++) regmap[hr]=-1;
 500 }
 501
 502 signed char get_reg(signed char regmap[],int r)
 503 {
 504   int hr;
 505   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap[hr]==r) return hr;
 506   return -1;
 507 }
 508
 509 // Find a register that is available for two consecutive cycles
 510 signed char get_reg2(signed char regmap1[],signed char regmap2[],int r)
 511 {
 512   int hr;
 513   for (hr=0;hr<HOST_REGS;hr++) if(hr!=EXCLUDE_REG&&regmap1[hr]==r&&regmap2[hr]==r) return hr;
 514   return -1;
 515 }
 516
 517 int count_free_regs(signed char regmap[])
 518 {
 519   int count=0;
 520   int hr;
 521   for(hr=0;hr<HOST_REGS;hr++)
 522   {
 523     if(hr!=EXCLUDE_REG) {
 524       if(regmap[hr]<0) count++;
 525     }
 526   }
 527   return count;
 528 }
 529
 530 void dirty_reg(struct regstat *cur,signed char reg)
 531 {
 532   int hr;
 533   if(!reg) return;
 534   for (hr=0;hr<HOST_REGS;hr++) {
 535     if((cur->regmap[hr]&63)==reg) {
 536       cur->dirty|=1<<hr;
 537     }
 538   }
 539 }
 540
 541 // If we dirty the lower half of a 64 bit register which is now being
 542 // sign-extended, we need to dump the upper half.
 543 // Note: Do this only after completion of the instruction, because
 544 // some instructions may need to read the full 64-bit value even if
 545 // overwriting it (eg SLTI, DSRA32).
 546 static void flush_dirty_uppers(struct regstat *cur)
 547 {
 548   int hr,reg;
 549   for (hr=0;hr<HOST_REGS;hr++) {
 550     if((cur->dirty>>hr)&1) {
 551       reg=cur->regmap[hr];
 552       if(reg>=64)
 553         if((cur->is32>>(reg&63))&1) cur->regmap[hr]=-1;
 554     }
 555   }
 556 }
 557
 558 void set_const(struct regstat *cur,signed char reg,uint64_t value)
 559 {
 560   int hr;
 561   if(!reg) return;
 562   for (hr=0;hr<HOST_REGS;hr++) {
 563     if(cur->regmap[hr]==reg) {
 564       cur->isconst|=1<<hr;
 565       cur->constmap[hr]=value;
 566     }
 567     else if((cur->regmap[hr]^64)==reg) {
 568       cur->isconst|=1<<hr;
 569       cur->constmap[hr]=value>>32;
 570     }
 571   }
 572 }
 573
 574 void clear_const(struct regstat *cur,signed char reg)
 575 {
 576   int hr;
 577   if(!reg) return;
 578   for (hr=0;hr<HOST_REGS;hr++) {
 579     if((cur->regmap[hr]&63)==reg) {
 580       cur->isconst&=~(1<<hr);
 581     }
 582   }
 583 }
 584
 585 int is_const(struct regstat *cur,signed char reg)
 586 {
 587   int hr;
 588   if(!reg) return 1;
 589   for (hr=0;hr<HOST_REGS;hr++) {
 590     if((cur->regmap[hr]&63)==reg) {
 591       return (cur->isconst>>hr)&1;
 592     }
 593   }
 594   return 0;
 595 }
 596 uint64_t get_const(struct regstat *cur,signed char reg)
 597 {
 598   int hr;
 599   if(!reg) return 0;
 600   for (hr=0;hr<HOST_REGS;hr++) {
 601     if(cur->regmap[hr]==reg) {
 602       return cur->constmap[hr];
 603     }
 604   }
 605   printf("Unknown constant in r%d\n",reg);
 606   exit(1);
 607 }
 608
 609 // Least soon needed registers
 610 // Look at the next ten instructions and see which registers
 611 // will be used.  Try not to reallocate these.
 612 void lsn(u_char hsn[], int i, int *preferred_reg)
 613 {
 614   int j;
 615   int b=-1;
 616   for(j=0;j<9;j++)
 617   {
 618     if(i+j>=slen) {
 619       j=slen-i-1;
 620       break;
 621     }
 622     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 623     {
 624       // Don't go past an unconditonal jump
 625       j++;
 626       break;
 627     }
 628   }
 629   for(;j>=0;j--)
 630   {
 631     if(rs1[i+j]) hsn[rs1[i+j]]=j;
 632     if(rs2[i+j]) hsn[rs2[i+j]]=j;
 633     if(rt1[i+j]) hsn[rt1[i+j]]=j;
 634     if(rt2[i+j]) hsn[rt2[i+j]]=j;
 635     if(itype[i+j]==STORE || itype[i+j]==STORELR) {
 636       // Stores can allocate zero
 637       hsn[rs1[i+j]]=j;
 638       hsn[rs2[i+j]]=j;
 639     }
 640     // On some architectures stores need invc_ptr
 641     #if defined(HOST_IMM8)
 642     if(itype[i+j]==STORE || itype[i+j]==STORELR || (opcode[i+j]&0x3b)==0x39 || (opcode[i+j]&0x3b)==0x3a) {
 643       hsn[INVCP]=j;
 644     }
 645     #endif
 646     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP||itype[i+j]==FJUMP))
 647     {
 648       hsn[CCREG]=j;
 649       b=j;
 650     }
 651   }
 652   if(b>=0)
 653   {
 654     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 655     {
 656       // Follow first branch
 657       int t=(ba[i+b]-start)>>2;
 658       j=7-b;if(t+j>=slen) j=slen-t-1;
 659       for(;j>=0;j--)
 660       {
 661         if(rs1[t+j]) if(hsn[rs1[t+j]]>j+b+2) hsn[rs1[t+j]]=j+b+2;
 662         if(rs2[t+j]) if(hsn[rs2[t+j]]>j+b+2) hsn[rs2[t+j]]=j+b+2;
 663         //if(rt1[t+j]) if(hsn[rt1[t+j]]>j+b+2) hsn[rt1[t+j]]=j+b+2;
 664         //if(rt2[t+j]) if(hsn[rt2[t+j]]>j+b+2) hsn[rt2[t+j]]=j+b+2;
 665       }
 666     }
 667     // TODO: preferred register based on backward branch
 668   }
 669   // Delay slot should preferably not overwrite branch conditions or cycle count
 670   if(i>0&&(itype[i-1]==RJUMP||itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP)) {
 671     if(rs1[i-1]) if(hsn[rs1[i-1]]>1) hsn[rs1[i-1]]=1;
 672     if(rs2[i-1]) if(hsn[rs2[i-1]]>1) hsn[rs2[i-1]]=1;
 673     hsn[CCREG]=1;
 674     // ...or hash tables
 675     hsn[RHASH]=1;
 676     hsn[RHTBL]=1;
 677   }
 678   // Coprocessor load/store needs FTEMP, even if not declared
 679   if(itype[i]==C1LS||itype[i]==C2LS) {
 680     hsn[FTEMP]=0;
 681   }
 682   // Load L/R also uses FTEMP as a temporary register
 683   if(itype[i]==LOADLR) {
 684     hsn[FTEMP]=0;
 685   }
 686   // Also SWL/SWR/SDL/SDR
 687   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) {
 688     hsn[FTEMP]=0;
 689   }
 690   // Don't remove the TLB registers either
 691   if(itype[i]==LOAD || itype[i]==LOADLR || itype[i]==STORE || itype[i]==STORELR || itype[i]==C1LS || itype[i]==C2LS) {
 692     hsn[TLREG]=0;
 693   }
 694   // Don't remove the miniht registers
 695   if(itype[i]==UJUMP||itype[i]==RJUMP)
 696   {
 697     hsn[RHASH]=0;
 698     hsn[RHTBL]=0;
 699   }
 700 }
 701
 702 // We only want to allocate registers if we're going to use them again soon
 703 int needed_again(int r, int i)
 704 {
 705   int j;
 706   int b=-1;
 707   int rn=10;
 708   int hr;
 709   u_char hsn[MAXREG+1];
 710   int preferred_reg;
 711
 712   memset(hsn,10,sizeof(hsn));
 713   lsn(hsn,i,&preferred_reg);
 714
 715   if(i>0&&(itype[i-1]==UJUMP||itype[i-1]==RJUMP||(source[i-1]>>16)==0x1000))
 716   {
 717     if(ba[i-1]<start || ba[i-1]>start+slen*4-4)
 718       return 0; // Don't need any registers if exiting the block
 719   }
 720   for(j=0;j<9;j++)
 721   {
 722     if(i+j>=slen) {
 723       j=slen-i-1;
 724       break;
 725     }
 726     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 727     {
 728       // Don't go past an unconditonal jump
 729       j++;
 730       break;
 731     }
 732     if(itype[i+j]==SYSCALL||itype[i+j]==HLECALL||itype[i+j]==INTCALL||((source[i+j]&0xfc00003f)==0x0d))
 733     {
 734       break;
 735     }
 736   }
 737   for(;j>=1;j--)
 738   {
 739     if(rs1[i+j]==r) rn=j;
 740     if(rs2[i+j]==r) rn=j;
 741     if((unneeded_reg[i+j]>>r)&1) rn=10;
 742     if(i+j>=0&&(itype[i+j]==UJUMP||itype[i+j]==CJUMP||itype[i+j]==SJUMP||itype[i+j]==FJUMP))
 743     {
 744       b=j;
 745     }
 746   }
 747   /*
 748   if(b>=0)
 749   {
 750     if(ba[i+b]>=start && ba[i+b]<(start+slen*4))
 751     {
 752       // Follow first branch
 753       int o=rn;
 754       int t=(ba[i+b]-start)>>2;
 755       j=7-b;if(t+j>=slen) j=slen-t-1;
 756       for(;j>=0;j--)
 757       {
 758         if(!((unneeded_reg[t+j]>>r)&1)) {
 759           if(rs1[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 760           if(rs2[t+j]==r) if(rn>j+b+2) rn=j+b+2;
 761         }
 762         else rn=o;
 763       }
 764     }
 765   }*/
 766   for(hr=0;hr<HOST_REGS;hr++) {
 767     if(hr!=EXCLUDE_REG) {
 768       if(rn<hsn[hr]) return 1;
 769     }
 770   }
 771   return 0;
 772 }
 773
 774 // Try to match register allocations at the end of a loop with those
 775 // at the beginning
 776 int loop_reg(int i, int r, int hr)
 777 {
 778   int j,k;
 779   for(j=0;j<9;j++)
 780   {
 781     if(i+j>=slen) {
 782       j=slen-i-1;
 783       break;
 784     }
 785     if(itype[i+j]==UJUMP||itype[i+j]==RJUMP||(source[i+j]>>16)==0x1000)
 786     {
 787       // Don't go past an unconditonal jump
 788       j++;
 789       break;
 790     }
 791   }
 792   k=0;
 793   if(i>0){
 794     if(itype[i-1]==UJUMP||itype[i-1]==CJUMP||itype[i-1]==SJUMP||itype[i-1]==FJUMP)
 795       k--;
 796   }
 797   for(;k<j;k++)
 798   {
 799     if(r<64&&((unneeded_reg[i+k]>>r)&1)) return hr;
 800     if(r>64&&((unneeded_reg_upper[i+k]>>r)&1)) return hr;
 801     if(i+k>=0&&(itype[i+k]==UJUMP||itype[i+k]==CJUMP||itype[i+k]==SJUMP||itype[i+k]==FJUMP))
 802     {
 803       if(ba[i+k]>=start && ba[i+k]<(start+i*4))
 804       {
 805         int t=(ba[i+k]-start)>>2;
 806         int reg=get_reg(regs[t].regmap_entry,r);
 807         if(reg>=0) return reg;
 808         //reg=get_reg(regs[t+1].regmap_entry,r);
 809         //if(reg>=0) return reg;
 810       }
 811     }
 812   }
 813   return hr;
 814 }
 815
 816
 817 // Allocate every register, preserving source/target regs
 818 void alloc_all(struct regstat *cur,int i)
 819 {
 820   int hr;
 821
 822   for(hr=0;hr<HOST_REGS;hr++) {
 823     if(hr!=EXCLUDE_REG) {
 824       if(((cur->regmap[hr]&63)!=rs1[i])&&((cur->regmap[hr]&63)!=rs2[i])&&
 825          ((cur->regmap[hr]&63)!=rt1[i])&&((cur->regmap[hr]&63)!=rt2[i]))
 826       {
 827         cur->regmap[hr]=-1;
 828         cur->dirty&=~(1<<hr);
 829       }
 830       // Don't need zeros
 831       if((cur->regmap[hr]&63)==0)
 832       {
 833         cur->regmap[hr]=-1;
 834         cur->dirty&=~(1<<hr);
 835       }
 836     }
 837   }
 838 }
 839
 840
 841 void div64(int64_t dividend,int64_t divisor)
 842 {
 843   lo=dividend/divisor;
 844   hi=dividend%divisor;
 845   //printf("TRACE: ddiv %8x%8x %8x%8x\n" ,(int)reg[HIREG],(int)(reg[HIREG]>>32)
 846   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 847 }
 848 void divu64(uint64_t dividend,uint64_t divisor)
 849 {
 850   lo=dividend/divisor;
 851   hi=dividend%divisor;
 852   //printf("TRACE: ddivu %8x%8x %8x%8x\n",(int)reg[HIREG],(int)(reg[HIREG]>>32)
 853   //                                     ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 854 }
 855
 856 void mult64(uint64_t m1,uint64_t m2)
 857 {
 858    unsigned long long int op1, op2, op3, op4;
 859    unsigned long long int result1, result2, result3, result4;
 860    unsigned long long int temp1, temp2, temp3, temp4;
 861    int sign = 0;
 862
 863    if (m1 < 0)
 864      {
 865     op2 = -m1;
 866     sign = 1 - sign;
 867      }
 868    else op2 = m1;
 869    if (m2 < 0)
 870      {
 871     op4 = -m2;
 872     sign = 1 - sign;
 873      }
 874    else op4 = m2;
 875
 876    op1 = op2 & 0xFFFFFFFF;
 877    op2 = (op2 >> 32) & 0xFFFFFFFF;
 878    op3 = op4 & 0xFFFFFFFF;
 879    op4 = (op4 >> 32) & 0xFFFFFFFF;
 880
 881    temp1 = op1 * op3;
 882    temp2 = (temp1 >> 32) + op1 * op4;
 883    temp3 = op2 * op3;
 884    temp4 = (temp3 >> 32) + op2 * op4;
 885
 886    result1 = temp1 & 0xFFFFFFFF;
 887    result2 = temp2 + (temp3 & 0xFFFFFFFF);
 888    result3 = (result2 >> 32) + temp4;
 889    result4 = (result3 >> 32);
 890
 891    lo = result1 | (result2 << 32);
 892    hi = (result3 & 0xFFFFFFFF) | (result4 << 32);
 893    if (sign)
 894      {
 895     hi = ~hi;
 896     if (!lo) hi++;
 897     else lo = ~lo + 1;
 898      }
 899 }
 900
 901 void multu64(uint64_t m1,uint64_t m2)
 902 {
 903    unsigned long long int op1, op2, op3, op4;
 904    unsigned long long int result1, result2, result3, result4;
 905    unsigned long long int temp1, temp2, temp3, temp4;
 906
 907    op1 = m1 & 0xFFFFFFFF;
 908    op2 = (m1 >> 32) & 0xFFFFFFFF;
 909    op3 = m2 & 0xFFFFFFFF;
 910    op4 = (m2 >> 32) & 0xFFFFFFFF;
 911
 912    temp1 = op1 * op3;
 913    temp2 = (temp1 >> 32) + op1 * op4;
 914    temp3 = op2 * op3;
 915    temp4 = (temp3 >> 32) + op2 * op4;
 916
 917    result1 = temp1 & 0xFFFFFFFF;
 918    result2 = temp2 + (temp3 & 0xFFFFFFFF);
 919    result3 = (result2 >> 32) + temp4;
 920    result4 = (result3 >> 32);
 921
 922    lo = result1 | (result2 << 32);
 923    hi = (result3 & 0xFFFFFFFF) | (result4 << 32);
 924
 925   //printf("TRACE: dmultu %8x%8x %8x%8x\n",(int)reg[HIREG],(int)(reg[HIREG]>>32)
 926   //                                      ,(int)reg[LOREG],(int)(reg[LOREG]>>32));
 927 }
 928
 929 uint64_t ldl_merge(uint64_t original,uint64_t loaded,u_int bits)
 930 {
 931   if(bits) {
 932     original<<=64-bits;
 933     original>>=64-bits;
 934     loaded<<=bits;
 935     original|=loaded;
 936   }
 937   else original=loaded;
 938   return original;
 939 }
 940 uint64_t ldr_merge(uint64_t original,uint64_t loaded,u_int bits)
 941 {
 942   if(bits^56) {
 943     original>>=64-(bits^56);
 944     original<<=64-(bits^56);
 945     loaded>>=bits^56;
 946     original|=loaded;
 947   }
 948   else original=loaded;
 949   return original;
 950 }
 951
 952 #ifdef __i386__
 953 #include "assem_x86.c"
 954 #endif
 955 #ifdef __x86_64__
 956 #include "assem_x64.c"
 957 #endif
 958 #ifdef __arm__
 959 #include "assem_arm.c"
 960 #endif
 961
 962 // Add virtual address mapping to linked list
 963 void ll_add(struct ll_entry **head,int vaddr,void *addr)
 964 {
 965   struct ll_entry *new_entry;
 966   new_entry=malloc(sizeof(struct ll_entry));
 967   assert(new_entry!=NULL);
 968   new_entry->vaddr=vaddr;
 969   new_entry->reg32=0;
 970   new_entry->addr=addr;
 971   new_entry->next=*head;
 972   *head=new_entry;
 973 }
 974
 975 // Add virtual address mapping for 32-bit compiled block
 976 void ll_add_32(struct ll_entry **head,int vaddr,u_int reg32,void *addr)
 977 {
 978   ll_add(head,vaddr,addr);
 979 #ifndef FORCE32
 980   (*head)->reg32=reg32;
 981 #endif
 982 }
 983
 984 // Check if an address is already compiled
 985 // but don't return addresses which are about to expire from the cache
 986 void *check_addr(u_int vaddr)
 987 {
 988   u_int *ht_bin=hash_table[((vaddr>>16)^vaddr)&0xFFFF];
 989   if(ht_bin[0]==vaddr) {
 990     if(((ht_bin[1]-MAX_OUTPUT_BLOCK_SIZE-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 991       if(isclean(ht_bin[1])) return (void *)ht_bin[1];
 992   }
 993   if(ht_bin[2]==vaddr) {
 994     if(((ht_bin[3]-MAX_OUTPUT_BLOCK_SIZE-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2)))
 995       if(isclean(ht_bin[3])) return (void *)ht_bin[3];
 996   }
 997   u_int page=get_page(vaddr);
 998   struct ll_entry *head;
 999   head=jump_in[page];
1000   while(head!=NULL) {
1001     if(head->vaddr==vaddr&&head->reg32==0) {
1002       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1003         // Update existing entry with current address
1004         if(ht_bin[0]==vaddr) {
1005           ht_bin[1]=(int)head->addr;
1006           return head->addr;
1007         }
1008         if(ht_bin[2]==vaddr) {
1009           ht_bin[3]=(int)head->addr;
1010           return head->addr;
1011         }
1012         // Insert into hash table with low priority.
1013         // Don't evict existing entries, as they are probably
1014         // addresses that are being accessed frequently.
1015         if(ht_bin[0]==-1) {
1016           ht_bin[1]=(int)head->addr;
1017           ht_bin[0]=vaddr;
1018         }else if(ht_bin[2]==-1) {
1019           ht_bin[3]=(int)head->addr;
1020           ht_bin[2]=vaddr;
1021         }
1022         return head->addr;
1023       }
1024     }
1025     head=head->next;
1026   }
1027   return 0;
1028 }
1029
1030 void remove_hash(int vaddr)
1031 {
1032   //printf("remove hash: %x\n",vaddr);
1033   int *ht_bin=hash_table[(((vaddr)>>16)^vaddr)&0xFFFF];
1034   if(ht_bin[2]==vaddr) {
1035     ht_bin[2]=ht_bin[3]=-1;
1036   }
1037   if(ht_bin[0]==vaddr) {
1038     ht_bin[0]=ht_bin[2];
1039     ht_bin[1]=ht_bin[3];
1040     ht_bin[2]=ht_bin[3]=-1;
1041   }
1042 }
1043
1044 void ll_remove_matching_addrs(struct ll_entry **head,int addr,int shift)
1045 {
1046   struct ll_entry *next;
1047   while(*head) {
1048     if(((u_int)((*head)->addr)>>shift)==(addr>>shift) ||
1049        ((u_int)((*head)->addr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift))
1050     {
1051       inv_debug("EXP: Remove pointer to %x (%x)\n",(int)(*head)->addr,(*head)->vaddr);
1052       remove_hash((*head)->vaddr);
1053       next=(*head)->next;
1054       free(*head);
1055       *head=next;
1056     }
1057     else
1058     {
1059       head=&((*head)->next);
1060     }
1061   }
1062 }
1063
1064 // Remove all entries from linked list
1065 void ll_clear(struct ll_entry **head)
1066 {
1067   struct ll_entry *cur;
1068   struct ll_entry *next;
1069   if(cur=*head) {
1070     *head=0;
1071     while(cur) {
1072       next=cur->next;
1073       free(cur);
1074       cur=next;
1075     }
1076   }
1077 }
1078
1079 // Dereference the pointers and remove if it matches
1080 void ll_kill_pointers(struct ll_entry *head,int addr,int shift)
1081 {
1082   u_int old_host_addr=0;
1083   while(head) {
1084     int ptr=get_pointer(head->addr);
1085     inv_debug("EXP: Lookup pointer to %x at %x (%x)\n",(int)ptr,(int)head->addr,head->vaddr);
1086     if(((ptr>>shift)==(addr>>shift)) ||
1087        (((ptr-MAX_OUTPUT_BLOCK_SIZE)>>shift)==(addr>>shift)))
1088     {
1089       inv_debug("EXP: Kill pointer at %x (%x)\n",(int)head->addr,head->vaddr);
1090       u_int host_addr=(u_int)kill_pointer(head->addr);
1091
1092       if((host_addr>>12)!=(old_host_addr>>12)) {
1093         #ifdef __arm__
1094         __clear_cache((void *)(old_host_addr&~0xfff),(void *)(old_host_addr|0xfff));
1095         #endif
1096         old_host_addr=host_addr;
1097       }
1098     }
1099     head=head->next;
1100   }
1101   #ifdef __arm__
1102   if (old_host_addr)
1103     __clear_cache((void *)(old_host_addr&~0xfff),(void *)(old_host_addr|0xfff));
1104   #endif
1105 }
1106
1107 // This is called when we write to a compiled block (see do_invstub)
1108 void invalidate_page(u_int page)
1109 {
1110   struct ll_entry *head;
1111   struct ll_entry *next;
1112   u_int old_host_addr=0;
1113   head=jump_in[page];
1114   jump_in[page]=0;
1115   while(head!=NULL) {
1116     inv_debug("INVALIDATE: %x\n",head->vaddr);
1117     remove_hash(head->vaddr);
1118     next=head->next;
1119     free(head);
1120     head=next;
1121   }
1122   head=jump_out[page];
1123   jump_out[page]=0;
1124   while(head!=NULL) {
1125     inv_debug("INVALIDATE: kill pointer to %x (%x)\n",head->vaddr,(int)head->addr);
1126     u_int host_addr=(u_int)kill_pointer(head->addr);
1127
1128     if((host_addr>>12)!=(old_host_addr>>12)) {
1129       #ifdef __arm__
1130       __clear_cache((void *)(old_host_addr&~0xfff),(void *)(old_host_addr|0xfff));
1131       #endif
1132       old_host_addr=host_addr;
1133     }
1134     next=head->next;
1135     free(head);
1136     head=next;
1137   }
1138   #ifdef __arm__
1139   if (old_host_addr)
1140     __clear_cache((void *)(old_host_addr&~0xfff),(void *)(old_host_addr|0xfff));
1141   #endif
1142 }
1143 void invalidate_block(u_int block)
1144 {
1145   u_int page=get_page(block<<12);
1146   u_int vpage=get_vpage(block<<12);
1147   inv_debug("INVALIDATE: %x (%d)\n",block<<12,page);
1148   //inv_debug("invalid_code[block]=%d\n",invalid_code[block]);
1149   u_int first,last;
1150   first=last=page;
1151   struct ll_entry *head;
1152   head=jump_dirty[vpage];
1153   //printf("page=%d vpage=%d\n",page,vpage);
1154   while(head!=NULL) {
1155     u_int start,end;
1156     if(vpage>2047||(head->vaddr>>12)==block) { // Ignore vaddr hash collision
1157       get_bounds((int)head->addr,&start,&end);
1158       //printf("start: %x end: %x\n",start,end);
1159       if(page<2048&&start>=0x80000000&&end<0x80000000+RAM_SIZE) {
1160         if(((start-(u_int)rdram)>>12)<=page&&((end-1-(u_int)rdram)>>12)>=page) {
1161           if((((start-(u_int)rdram)>>12)&2047)<first) first=((start-(u_int)rdram)>>12)&2047;
1162           if((((end-1-(u_int)rdram)>>12)&2047)>last) last=((end-1-(u_int)rdram)>>12)&2047;
1163         }
1164       }
1165 #ifndef DISABLE_TLB
1166       if(page<2048&&(signed int)start>=(signed int)0xC0000000&&(signed int)end>=(signed int)0xC0000000) {
1167         if(((start+memory_map[start>>12]-(u_int)rdram)>>12)<=page&&((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)>=page) {
1168           if((((start+memory_map[start>>12]-(u_int)rdram)>>12)&2047)<first) first=((start+memory_map[start>>12]-(u_int)rdram)>>12)&2047;
1169           if((((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)&2047)>last) last=((end-1+memory_map[(end-1)>>12]-(u_int)rdram)>>12)&2047;
1170         }
1171       }
1172 #endif
1173     }
1174     head=head->next;
1175   }
1176   //printf("first=%d last=%d\n",first,last);
1177   invalidate_page(page);
1178   assert(first+5>page); // NB: this assumes MAXBLOCK<=4096 (4 pages)
1179   assert(last<page+5);
1180   // Invalidate the adjacent pages if a block crosses a 4K boundary
1181   while(first<page) {
1182     invalidate_page(first);
1183     first++;
1184   }
1185   for(first=page+1;first<last;first++) {
1186     invalidate_page(first);
1187   }
1188
1189   // Don't trap writes
1190   invalid_code[block]=1;
1191 #ifndef DISABLE_TLB
1192   // If there is a valid TLB entry for this page, remove write protect
1193   if(tlb_LUT_w[block]) {
1194     assert(tlb_LUT_r[block]==tlb_LUT_w[block]);
1195     // CHECK: Is this right?
1196     memory_map[block]=((tlb_LUT_w[block]&0xFFFFF000)-(block<<12)+(unsigned int)rdram-0x80000000)>>2;
1197     u_int real_block=tlb_LUT_w[block]>>12;
1198     invalid_code[real_block]=1;
1199     if(real_block>=0x80000&&real_block<0x80800) memory_map[real_block]=((u_int)rdram-0x80000000)>>2;
1200   }
1201   else if(block>=0x80000&&block<0x80800) memory_map[block]=((u_int)rdram-0x80000000)>>2;
1202 #endif
1203
1204   #ifdef USE_MINI_HT
1205   memset(mini_ht,-1,sizeof(mini_ht));
1206   #endif
1207 }
1208 void invalidate_addr(u_int addr)
1209 {
1210   invalidate_block(addr>>12);
1211 }
1212 void invalidate_all_pages()
1213 {
1214   u_int page,n;
1215   for(page=0;page<4096;page++)
1216     invalidate_page(page);
1217   for(page=0;page<1048576;page++)
1218     if(!invalid_code[page]) {
1219       restore_candidate[(page&2047)>>3]|=1<<(page&7);
1220       restore_candidate[((page&2047)>>3)+256]|=1<<(page&7);
1221     }
1222   #ifdef __arm__
1223   __clear_cache((void *)BASE_ADDR,(void *)BASE_ADDR+(1<<TARGET_SIZE_2));
1224   #endif
1225   #ifdef USE_MINI_HT
1226   memset(mini_ht,-1,sizeof(mini_ht));
1227   #endif
1228   #ifndef DISABLE_TLB
1229   // TLB
1230   for(page=0;page<0x100000;page++) {
1231     if(tlb_LUT_r[page]) {
1232       memory_map[page]=((tlb_LUT_r[page]&0xFFFFF000)-(page<<12)+(unsigned int)rdram-0x80000000)>>2;
1233       if(!tlb_LUT_w[page]||!invalid_code[page])
1234         memory_map[page]|=0x40000000; // Write protect
1235     }
1236     else memory_map[page]=-1;
1237     if(page==0x80000) page=0xC0000;
1238   }
1239   tlb_hacks();
1240   #endif
1241 }
1242
1243 // Add an entry to jump_out after making a link
1244 void add_link(u_int vaddr,void *src)
1245 {
1246   u_int page=get_page(vaddr);
1247   inv_debug("add_link: %x -> %x (%d)\n",(int)src,vaddr,page);
1248   ll_add(jump_out+page,vaddr,src);
1249   //int ptr=get_pointer(src);
1250   //inv_debug("add_link: Pointer is to %x\n",(int)ptr);
1251 }
1252
1253 // If a code block was found to be unmodified (bit was set in
1254 // restore_candidate) and it remains unmodified (bit is clear
1255 // in invalid_code) then move the entries for that 4K page from
1256 // the dirty list to the clean list.
1257 void clean_blocks(u_int page)
1258 {
1259   struct ll_entry *head;
1260   inv_debug("INV: clean_blocks page=%d\n",page);
1261   head=jump_dirty[page];
1262   while(head!=NULL) {
1263     if(!invalid_code[head->vaddr>>12]) {
1264       // Don't restore blocks which are about to expire from the cache
1265       if((((u_int)head->addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1266         u_int start,end;
1267         if(verify_dirty((int)head->addr)) {
1268           //printf("Possibly Restore %x (%x)\n",head->vaddr, (int)head->addr);
1269           u_int i;
1270           u_int inv=0;
1271           get_bounds((int)head->addr,&start,&end);
1272           if(start-(u_int)rdram<RAM_SIZE) {
1273             for(i=(start-(u_int)rdram+0x80000000)>>12;i<=(end-1-(u_int)rdram+0x80000000)>>12;i++) {
1274               inv|=invalid_code[i];
1275             }
1276           }
1277           if((signed int)head->vaddr>=(signed int)0xC0000000) {
1278             u_int addr = (head->vaddr+(memory_map[head->vaddr>>12]<<2));
1279             //printf("addr=%x start=%x end=%x\n",addr,start,end);
1280             if(addr<start||addr>=end) inv=1;
1281           }
1282           else if((signed int)head->vaddr>=(signed int)0x80000000+RAM_SIZE) {
1283             inv=1;
1284           }
1285           if(!inv) {
1286             void * clean_addr=(void *)get_clean_addr((int)head->addr);
1287             if((((u_int)clean_addr-(u_int)out)<<(32-TARGET_SIZE_2))>0x60000000+(MAX_OUTPUT_BLOCK_SIZE<<(32-TARGET_SIZE_2))) {
1288               u_int ppage=page;
1289 #ifndef DISABLE_TLB
1290               if(page<2048&&tlb_LUT_r[head->vaddr>>12]) ppage=(tlb_LUT_r[head->vaddr>>12]^0x80000000)>>12;
1291 #endif
1292               inv_debug("INV: Restored %x (%x/%x)\n",head->vaddr, (int)head->addr, (int)clean_addr);
1293               //printf("page=%x, addr=%x\n",page,head->vaddr);
1294               //assert(head->vaddr>>12==(page|0x80000));
1295               ll_add_32(jump_in+ppage,head->vaddr,head->reg32,clean_addr);
1296               int *ht_bin=hash_table[((head->vaddr>>16)^head->vaddr)&0xFFFF];
1297               if(!head->reg32) {
1298                 if(ht_bin[0]==head->vaddr) {
1299                   ht_bin[1]=(int)clean_addr; // Replace existing entry
1300                 }
1301                 if(ht_bin[2]==head->vaddr) {
1302                   ht_bin[3]=(int)clean_addr; // Replace existing entry
1303                 }
1304               }
1305             }
1306           }
1307         }
1308       }
1309     }
1310     head=head->next;
1311   }
1312 }
1313
1314
1315 void mov_alloc(struct regstat *current,int i)
1316 {
1317   // Note: Don't need to actually alloc the source registers
1318   if((~current->is32>>rs1[i])&1) {
1319     //alloc_reg64(current,i,rs1[i]);
1320     alloc_reg64(current,i,rt1[i]);
1321     current->is32&=~(1LL<<rt1[i]);
1322   } else {
1323     //alloc_reg(current,i,rs1[i]);
1324     alloc_reg(current,i,rt1[i]);
1325     current->is32|=(1LL<<rt1[i]);
1326   }
1327   clear_const(current,rs1[i]);
1328   clear_const(current,rt1[i]);
1329   dirty_reg(current,rt1[i]);
1330 }
1331
1332 void shiftimm_alloc(struct regstat *current,int i)
1333 {
1334   clear_const(current,rs1[i]);
1335   clear_const(current,rt1[i]);
1336   if(opcode2[i]<=0x3) // SLL/SRL/SRA
1337   {
1338     if(rt1[i]) {
1339       if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1340       else lt1[i]=rs1[i];
1341       alloc_reg(current,i,rt1[i]);
1342       current->is32|=1LL<<rt1[i];
1343       dirty_reg(current,rt1[i]);
1344     }
1345   }
1346   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
1347   {
1348     if(rt1[i]) {
1349       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1350       alloc_reg64(current,i,rt1[i]);
1351       current->is32&=~(1LL<<rt1[i]);
1352       dirty_reg(current,rt1[i]);
1353     }
1354   }
1355   if(opcode2[i]==0x3c) // DSLL32
1356   {
1357     if(rt1[i]) {
1358       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1359       alloc_reg64(current,i,rt1[i]);
1360       current->is32&=~(1LL<<rt1[i]);
1361       dirty_reg(current,rt1[i]);
1362     }
1363   }
1364   if(opcode2[i]==0x3e) // DSRL32
1365   {
1366     if(rt1[i]) {
1367       alloc_reg64(current,i,rs1[i]);
1368       if(imm[i]==32) {
1369         alloc_reg64(current,i,rt1[i]);
1370         current->is32&=~(1LL<<rt1[i]);
1371       } else {
1372         alloc_reg(current,i,rt1[i]);
1373         current->is32|=1LL<<rt1[i];
1374       }
1375       dirty_reg(current,rt1[i]);
1376     }
1377   }
1378   if(opcode2[i]==0x3f) // DSRA32
1379   {
1380     if(rt1[i]) {
1381       alloc_reg64(current,i,rs1[i]);
1382       alloc_reg(current,i,rt1[i]);
1383       current->is32|=1LL<<rt1[i];
1384       dirty_reg(current,rt1[i]);
1385     }
1386   }
1387 }
1388
1389 void shift_alloc(struct regstat *current,int i)
1390 {
1391   if(rt1[i]) {
1392     if(opcode2[i]<=0x07) // SLLV/SRLV/SRAV
1393     {
1394       if(rs1[i]) alloc_reg(current,i,rs1[i]);
1395       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1396       alloc_reg(current,i,rt1[i]);
1397       if(rt1[i]==rs2[i]) alloc_reg_temp(current,i,-1);
1398       current->is32|=1LL<<rt1[i];
1399     } else { // DSLLV/DSRLV/DSRAV
1400       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1401       if(rs2[i]) alloc_reg(current,i,rs2[i]);
1402       alloc_reg64(current,i,rt1[i]);
1403       current->is32&=~(1LL<<rt1[i]);
1404       if(opcode2[i]==0x16||opcode2[i]==0x17) // DSRLV and DSRAV need a temporary register
1405         alloc_reg_temp(current,i,-1);
1406     }
1407     clear_const(current,rs1[i]);
1408     clear_const(current,rs2[i]);
1409     clear_const(current,rt1[i]);
1410     dirty_reg(current,rt1[i]);
1411   }
1412 }
1413
1414 void alu_alloc(struct regstat *current,int i)
1415 {
1416   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
1417     if(rt1[i]) {
1418       if(rs1[i]&&rs2[i]) {
1419         alloc_reg(current,i,rs1[i]);
1420         alloc_reg(current,i,rs2[i]);
1421       }
1422       else {
1423         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1424         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1425       }
1426       alloc_reg(current,i,rt1[i]);
1427     }
1428     current->is32|=1LL<<rt1[i];
1429   }
1430   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
1431     if(rt1[i]) {
1432       if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1433       {
1434         alloc_reg64(current,i,rs1[i]);
1435         alloc_reg64(current,i,rs2[i]);
1436         alloc_reg(current,i,rt1[i]);
1437       } else {
1438         alloc_reg(current,i,rs1[i]);
1439         alloc_reg(current,i,rs2[i]);
1440         alloc_reg(current,i,rt1[i]);
1441       }
1442     }
1443     current->is32|=1LL<<rt1[i];
1444   }
1445   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
1446     if(rt1[i]) {
1447       if(rs1[i]&&rs2[i]) {
1448         alloc_reg(current,i,rs1[i]);
1449         alloc_reg(current,i,rs2[i]);
1450       }
1451       else
1452       {
1453         if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1454         if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg(current,i,rs2[i]);
1455       }
1456       alloc_reg(current,i,rt1[i]);
1457       if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1458       {
1459         if(!((current->uu>>rt1[i])&1)) {
1460           alloc_reg64(current,i,rt1[i]);
1461         }
1462         if(get_reg(current->regmap,rt1[i]|64)>=0) {
1463           if(rs1[i]&&rs2[i]) {
1464             alloc_reg64(current,i,rs1[i]);
1465             alloc_reg64(current,i,rs2[i]);
1466           }
1467           else
1468           {
1469             // Is is really worth it to keep 64-bit values in registers?
1470             #ifdef NATIVE_64BIT
1471             if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg64(current,i,rs1[i]);
1472             if(rs2[i]&&needed_again(rs2[i],i)) alloc_reg64(current,i,rs2[i]);
1473             #endif
1474           }
1475         }
1476         current->is32&=~(1LL<<rt1[i]);
1477       } else {
1478         current->is32|=1LL<<rt1[i];
1479       }
1480     }
1481   }
1482   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
1483     if(rt1[i]) {
1484       if(rs1[i]&&rs2[i]) {
1485         if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1486           alloc_reg64(current,i,rs1[i]);
1487           alloc_reg64(current,i,rs2[i]);
1488           alloc_reg64(current,i,rt1[i]);
1489         } else {
1490           alloc_reg(current,i,rs1[i]);
1491           alloc_reg(current,i,rs2[i]);
1492           alloc_reg(current,i,rt1[i]);
1493         }
1494       }
1495       else {
1496         alloc_reg(current,i,rt1[i]);
1497         if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1498           // DADD used as move, or zeroing
1499           // If we have a 64-bit source, then make the target 64 bits too
1500           if(rs1[i]&&!((current->is32>>rs1[i])&1)) {
1501             if(get_reg(current->regmap,rs1[i])>=0) alloc_reg64(current,i,rs1[i]);
1502             alloc_reg64(current,i,rt1[i]);
1503           } else if(rs2[i]&&!((current->is32>>rs2[i])&1)) {
1504             if(get_reg(current->regmap,rs2[i])>=0) alloc_reg64(current,i,rs2[i]);
1505             alloc_reg64(current,i,rt1[i]);
1506           }
1507           if(opcode2[i]>=0x2e&&rs2[i]) {
1508             // DSUB used as negation - 64-bit result
1509             // If we have a 32-bit register, extend it to 64 bits
1510             if(get_reg(current->regmap,rs2[i])>=0) alloc_reg64(current,i,rs2[i]);
1511             alloc_reg64(current,i,rt1[i]);
1512           }
1513         }
1514       }
1515       if(rs1[i]&&rs2[i]) {
1516         current->is32&=~(1LL<<rt1[i]);
1517       } else if(rs1[i]) {
1518         current->is32&=~(1LL<<rt1[i]);
1519         if((current->is32>>rs1[i])&1)
1520           current->is32|=1LL<<rt1[i];
1521       } else if(rs2[i]) {
1522         current->is32&=~(1LL<<rt1[i]);
1523         if((current->is32>>rs2[i])&1)
1524           current->is32|=1LL<<rt1[i];
1525       } else {
1526         current->is32|=1LL<<rt1[i];
1527       }
1528     }
1529   }
1530   clear_const(current,rs1[i]);
1531   clear_const(current,rs2[i]);
1532   clear_const(current,rt1[i]);
1533   dirty_reg(current,rt1[i]);
1534 }
1535
1536 void imm16_alloc(struct regstat *current,int i)
1537 {
1538   if(rs1[i]&&needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1539   else lt1[i]=rs1[i];
1540   if(rt1[i]) alloc_reg(current,i,rt1[i]);
1541   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
1542     current->is32&=~(1LL<<rt1[i]);
1543     if(!((current->uu>>rt1[i])&1)||get_reg(current->regmap,rt1[i]|64)>=0) {
1544       // TODO: Could preserve the 32-bit flag if the immediate is zero
1545       alloc_reg64(current,i,rt1[i]);
1546       alloc_reg64(current,i,rs1[i]);
1547     }
1548     clear_const(current,rs1[i]);
1549     clear_const(current,rt1[i]);
1550   }
1551   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
1552     if((~current->is32>>rs1[i])&1) alloc_reg64(current,i,rs1[i]);
1553     current->is32|=1LL<<rt1[i];
1554     clear_const(current,rs1[i]);
1555     clear_const(current,rt1[i]);
1556   }
1557   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
1558     if(((~current->is32>>rs1[i])&1)&&opcode[i]>0x0c) {
1559       if(rs1[i]!=rt1[i]) {
1560         if(needed_again(rs1[i],i)) alloc_reg64(current,i,rs1[i]);
1561         alloc_reg64(current,i,rt1[i]);
1562         current->is32&=~(1LL<<rt1[i]);
1563       }
1564     }
1565     else current->is32|=1LL<<rt1[i]; // ANDI clears upper bits
1566     if(is_const(current,rs1[i])) {
1567       int v=get_const(current,rs1[i]);
1568       if(opcode[i]==0x0c) set_const(current,rt1[i],v&imm[i]);
1569       if(opcode[i]==0x0d) set_const(current,rt1[i],v|imm[i]);
1570       if(opcode[i]==0x0e) set_const(current,rt1[i],v^imm[i]);
1571     }
1572     else clear_const(current,rt1[i]);
1573   }
1574   else if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
1575     if(is_const(current,rs1[i])) {
1576       int v=get_const(current,rs1[i]);
1577       set_const(current,rt1[i],v+imm[i]);
1578     }
1579     else clear_const(current,rt1[i]);
1580     current->is32|=1LL<<rt1[i];
1581   }
1582   else {
1583     set_const(current,rt1[i],((long long)((short)imm[i]))<<16); // LUI
1584     current->is32|=1LL<<rt1[i];
1585   }
1586   dirty_reg(current,rt1[i]);
1587 }
1588
1589 void load_alloc(struct regstat *current,int i)
1590 {
1591   clear_const(current,rt1[i]);
1592   //if(rs1[i]!=rt1[i]&&needed_again(rs1[i],i)) clear_const(current,rs1[i]); // Does this help or hurt?
1593   if(!rs1[i]) current->u&=~1LL; // Allow allocating r0 if it's the source register
1594   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1595   if(rt1[i]) {
1596     alloc_reg(current,i,rt1[i]);
1597     if(opcode[i]==0x27||opcode[i]==0x37) // LWU/LD
1598     {
1599       current->is32&=~(1LL<<rt1[i]);
1600       alloc_reg64(current,i,rt1[i]);
1601     }
1602     else if(opcode[i]==0x1A||opcode[i]==0x1B) // LDL/LDR
1603     {
1604       current->is32&=~(1LL<<rt1[i]);
1605       alloc_reg64(current,i,rt1[i]);
1606       alloc_all(current,i);
1607       alloc_reg64(current,i,FTEMP);
1608     }
1609     else current->is32|=1LL<<rt1[i];
1610     dirty_reg(current,rt1[i]);
1611     // If using TLB, need a register for pointer to the mapping table
1612     if(using_tlb) alloc_reg(current,i,TLREG);
1613     // LWL/LWR need a temporary register for the old value
1614     if(opcode[i]==0x22||opcode[i]==0x26)
1615     {
1616       alloc_reg(current,i,FTEMP);
1617       alloc_reg_temp(current,i,-1);
1618     }
1619   }
1620   else
1621   {
1622     // Load to r0 (dummy load)
1623     // but we still need a register to calculate the address
1624     alloc_reg_temp(current,i,-1);
1625   }
1626 }
1627
1628 void store_alloc(struct regstat *current,int i)
1629 {
1630   clear_const(current,rs2[i]);
1631   if(!(rs2[i])) current->u&=~1LL; // Allow allocating r0 if necessary
1632   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1633   alloc_reg(current,i,rs2[i]);
1634   if(opcode[i]==0x2c||opcode[i]==0x2d||opcode[i]==0x3f) { // 64-bit SDL/SDR/SD
1635     alloc_reg64(current,i,rs2[i]);
1636     if(rs2[i]) alloc_reg(current,i,FTEMP);
1637   }
1638   // If using TLB, need a register for pointer to the mapping table
1639   if(using_tlb) alloc_reg(current,i,TLREG);
1640   #if defined(HOST_IMM8)
1641   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1642   else alloc_reg(current,i,INVCP);
1643   #endif
1644   if(opcode[i]==0x2a||opcode[i]==0x2e||opcode[i]==0x2c||opcode[i]==0x2d) { // SWL/SWL/SDL/SDR
1645     alloc_reg(current,i,FTEMP);
1646   }
1647   // We need a temporary register for address generation
1648   alloc_reg_temp(current,i,-1);
1649 }
1650
1651 void c1ls_alloc(struct regstat *current,int i)
1652 {
1653   //clear_const(current,rs1[i]); // FIXME
1654   clear_const(current,rt1[i]);
1655   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1656   alloc_reg(current,i,CSREG); // Status
1657   alloc_reg(current,i,FTEMP);
1658   if(opcode[i]==0x35||opcode[i]==0x3d) { // 64-bit LDC1/SDC1
1659     alloc_reg64(current,i,FTEMP);
1660   }
1661   // If using TLB, need a register for pointer to the mapping table
1662   if(using_tlb) alloc_reg(current,i,TLREG);
1663   #if defined(HOST_IMM8)
1664   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1665   else if((opcode[i]&0x3b)==0x39) // SWC1/SDC1
1666     alloc_reg(current,i,INVCP);
1667   #endif
1668   // We need a temporary register for address generation
1669   alloc_reg_temp(current,i,-1);
1670 }
1671
1672 void c2ls_alloc(struct regstat *current,int i)
1673 {
1674   clear_const(current,rt1[i]);
1675   if(needed_again(rs1[i],i)) alloc_reg(current,i,rs1[i]);
1676   alloc_reg(current,i,FTEMP);
1677   // If using TLB, need a register for pointer to the mapping table
1678   if(using_tlb) alloc_reg(current,i,TLREG);
1679   #if defined(HOST_IMM8)
1680   // On CPUs without 32-bit immediates we need a pointer to invalid_code
1681   else if((opcode[i]&0x3b)==0x3a) // SWC2/SDC2
1682     alloc_reg(current,i,INVCP);
1683   #endif
1684   // We need a temporary register for address generation
1685   alloc_reg_temp(current,i,-1);
1686 }
1687
1688 #ifndef multdiv_alloc
1689 void multdiv_alloc(struct regstat *current,int i)
1690 {
1691   //  case 0x18: MULT
1692   //  case 0x19: MULTU
1693   //  case 0x1A: DIV
1694   //  case 0x1B: DIVU
1695   //  case 0x1C: DMULT
1696   //  case 0x1D: DMULTU
1697   //  case 0x1E: DDIV
1698   //  case 0x1F: DDIVU
1699   clear_const(current,rs1[i]);
1700   clear_const(current,rs2[i]);
1701   if(rs1[i]&&rs2[i])
1702   {
1703     if((opcode2[i]&4)==0) // 32-bit
1704     {
1705       current->u&=~(1LL<<HIREG);
1706       current->u&=~(1LL<<LOREG);
1707       alloc_reg(current,i,HIREG);
1708       alloc_reg(current,i,LOREG);
1709       alloc_reg(current,i,rs1[i]);
1710       alloc_reg(current,i,rs2[i]);
1711       current->is32|=1LL<<HIREG;
1712       current->is32|=1LL<<LOREG;
1713       dirty_reg(current,HIREG);
1714       dirty_reg(current,LOREG);
1715     }
1716     else // 64-bit
1717     {
1718       current->u&=~(1LL<<HIREG);
1719       current->u&=~(1LL<<LOREG);
1720       current->uu&=~(1LL<<HIREG);
1721       current->uu&=~(1LL<<LOREG);
1722       alloc_reg64(current,i,HIREG);
1723       //if(HOST_REGS>10) alloc_reg64(current,i,LOREG);
1724       alloc_reg64(current,i,rs1[i]);
1725       alloc_reg64(current,i,rs2[i]);
1726       alloc_all(current,i);
1727       current->is32&=~(1LL<<HIREG);
1728       current->is32&=~(1LL<<LOREG);
1729       dirty_reg(current,HIREG);
1730       dirty_reg(current,LOREG);
1731     }
1732   }
1733   else
1734   {
1735     // Multiply by zero is zero.
1736     // MIPS does not have a divide by zero exception.
1737     // The result is undefined, we return zero.
1738     alloc_reg(current,i,HIREG);
1739     alloc_reg(current,i,LOREG);
1740     current->is32|=1LL<<HIREG;
1741     current->is32|=1LL<<LOREG;
1742     dirty_reg(current,HIREG);
1743     dirty_reg(current,LOREG);
1744   }
1745 }
1746 #endif
1747
1748 void cop0_alloc(struct regstat *current,int i)
1749 {
1750   if(opcode2[i]==0) // MFC0
1751   {
1752     if(rt1[i]) {
1753       clear_const(current,rt1[i]);
1754       alloc_all(current,i);
1755       alloc_reg(current,i,rt1[i]);
1756       current->is32|=1LL<<rt1[i];
1757       dirty_reg(current,rt1[i]);
1758     }
1759   }
1760   else if(opcode2[i]==4) // MTC0
1761   {
1762     if(rs1[i]){
1763       clear_const(current,rs1[i]);
1764       alloc_reg(current,i,rs1[i]);
1765       alloc_all(current,i);
1766     }
1767     else {
1768       alloc_all(current,i); // FIXME: Keep r0
1769       current->u&=~1LL;
1770       alloc_reg(current,i,0);
1771     }
1772   }
1773   else
1774   {
1775     // TLBR/TLBWI/TLBWR/TLBP/ERET
1776     assert(opcode2[i]==0x10);
1777     alloc_all(current,i);
1778   }
1779 }
1780
1781 void cop1_alloc(struct regstat *current,int i)
1782 {
1783   alloc_reg(current,i,CSREG); // Load status
1784   if(opcode2[i]<3) // MFC1/DMFC1/CFC1
1785   {
1786     assert(rt1[i]);
1787     clear_const(current,rt1[i]);
1788     if(opcode2[i]==1) {
1789       alloc_reg64(current,i,rt1[i]); // DMFC1
1790       current->is32&=~(1LL<<rt1[i]);
1791     }else{
1792       alloc_reg(current,i,rt1[i]); // MFC1/CFC1
1793       current->is32|=1LL<<rt1[i];
1794     }
1795     dirty_reg(current,rt1[i]);
1796     alloc_reg_temp(current,i,-1);
1797   }
1798   else if(opcode2[i]>3) // MTC1/DMTC1/CTC1
1799   {
1800     if(rs1[i]){
1801       clear_const(current,rs1[i]);
1802       if(opcode2[i]==5)
1803         alloc_reg64(current,i,rs1[i]); // DMTC1
1804       else
1805         alloc_reg(current,i,rs1[i]); // MTC1/CTC1
1806       alloc_reg_temp(current,i,-1);
1807     }
1808     else {
1809       current->u&=~1LL;
1810       alloc_reg(current,i,0);
1811       alloc_reg_temp(current,i,-1);
1812     }
1813   }
1814 }
1815 void fconv_alloc(struct regstat *current,int i)
1816 {
1817   alloc_reg(current,i,CSREG); // Load status
1818   alloc_reg_temp(current,i,-1);
1819 }
1820 void float_alloc(struct regstat *current,int i)
1821 {
1822   alloc_reg(current,i,CSREG); // Load status
1823   alloc_reg_temp(current,i,-1);
1824 }
1825 void c2op_alloc(struct regstat *current,int i)
1826 {
1827   alloc_reg_temp(current,i,-1);
1828 }
1829 void fcomp_alloc(struct regstat *current,int i)
1830 {
1831   alloc_reg(current,i,CSREG); // Load status
1832   alloc_reg(current,i,FSREG); // Load flags
1833   dirty_reg(current,FSREG); // Flag will be modified
1834   alloc_reg_temp(current,i,-1);
1835 }
1836
1837 void syscall_alloc(struct regstat *current,int i)
1838 {
1839   alloc_cc(current,i);
1840   dirty_reg(current,CCREG);
1841   alloc_all(current,i);
1842   current->isconst=0;
1843 }
1844
1845 void delayslot_alloc(struct regstat *current,int i)
1846 {
1847   switch(itype[i]) {
1848     case UJUMP:
1849     case CJUMP:
1850     case SJUMP:
1851     case RJUMP:
1852     case FJUMP:
1853     case SYSCALL:
1854     case HLECALL:
1855     case SPAN:
1856       assem_debug("jump in the delay slot.  this shouldn't happen.\n");//exit(1);
1857       printf("Disabled speculative precompilation\n");
1858       stop_after_jal=1;
1859       break;
1860     case IMM16:
1861       imm16_alloc(current,i);
1862       break;
1863     case LOAD:
1864     case LOADLR:
1865       load_alloc(current,i);
1866       break;
1867     case STORE:
1868     case STORELR:
1869       store_alloc(current,i);
1870       break;
1871     case ALU:
1872       alu_alloc(current,i);
1873       break;
1874     case SHIFT:
1875       shift_alloc(current,i);
1876       break;
1877     case MULTDIV:
1878       multdiv_alloc(current,i);
1879       break;
1880     case SHIFTIMM:
1881       shiftimm_alloc(current,i);
1882       break;
1883     case MOV:
1884       mov_alloc(current,i);
1885       break;
1886     case COP0:
1887       cop0_alloc(current,i);
1888       break;
1889     case COP1:
1890     case COP2:
1891       cop1_alloc(current,i);
1892       break;
1893     case C1LS:
1894       c1ls_alloc(current,i);
1895       break;
1896     case C2LS:
1897       c2ls_alloc(current,i);
1898       break;
1899     case FCONV:
1900       fconv_alloc(current,i);
1901       break;
1902     case FLOAT:
1903       float_alloc(current,i);
1904       break;
1905     case FCOMP:
1906       fcomp_alloc(current,i);
1907       break;
1908     case C2OP:
1909       c2op_alloc(current,i);
1910       break;
1911   }
1912 }
1913
1914 // Special case where a branch and delay slot span two pages in virtual memory
1915 static void pagespan_alloc(struct regstat *current,int i)
1916 {
1917   current->isconst=0;
1918   current->wasconst=0;
1919   regs[i].wasconst=0;
1920   alloc_all(current,i);
1921   alloc_cc(current,i);
1922   dirty_reg(current,CCREG);
1923   if(opcode[i]==3) // JAL
1924   {
1925     alloc_reg(current,i,31);
1926     dirty_reg(current,31);
1927   }
1928   if(opcode[i]==0&&(opcode2[i]&0x3E)==8) // JR/JALR
1929   {
1930     alloc_reg(current,i,rs1[i]);
1931     if (rt1[i]!=0) {
1932       alloc_reg(current,i,rt1[i]);
1933       dirty_reg(current,rt1[i]);
1934     }
1935   }
1936   if((opcode[i]&0x2E)==4) // BEQ/BNE/BEQL/BNEL
1937   {
1938     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1939     if(rs2[i]) alloc_reg(current,i,rs2[i]);
1940     if(!((current->is32>>rs1[i])&(current->is32>>rs2[i])&1))
1941     {
1942       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1943       if(rs2[i]) alloc_reg64(current,i,rs2[i]);
1944     }
1945   }
1946   else
1947   if((opcode[i]&0x2E)==6) // BLEZ/BGTZ/BLEZL/BGTZL
1948   {
1949     if(rs1[i]) alloc_reg(current,i,rs1[i]);
1950     if(!((current->is32>>rs1[i])&1))
1951     {
1952       if(rs1[i]) alloc_reg64(current,i,rs1[i]);
1953     }
1954   }
1955   else
1956   if(opcode[i]==0x11) // BC1
1957   {
1958     alloc_reg(current,i,FSREG);
1959     alloc_reg(current,i,CSREG);
1960   }
1961   //else ...
1962 }
1963
1964 add_stub(int type,int addr,int retaddr,int a,int b,int c,int d,int e)
1965 {
1966   stubs[stubcount][0]=type;
1967   stubs[stubcount][1]=addr;
1968   stubs[stubcount][2]=retaddr;
1969   stubs[stubcount][3]=a;
1970   stubs[stubcount][4]=b;
1971   stubs[stubcount][5]=c;
1972   stubs[stubcount][6]=d;
1973   stubs[stubcount][7]=e;
1974   stubcount++;
1975 }
1976
1977 // Write out a single register
1978 void wb_register(signed char r,signed char regmap[],uint64_t dirty,uint64_t is32)
1979 {
1980   int hr;
1981   for(hr=0;hr<HOST_REGS;hr++) {
1982     if(hr!=EXCLUDE_REG) {
1983       if((regmap[hr]&63)==r) {
1984         if((dirty>>hr)&1) {
1985           if(regmap[hr]<64) {
1986             emit_storereg(r,hr);
1987 #ifndef FORCE32
1988             if((is32>>regmap[hr])&1) {
1989               emit_sarimm(hr,31,hr);
1990               emit_storereg(r|64,hr);
1991             }
1992 #endif
1993           }else{
1994             emit_storereg(r|64,hr);
1995           }
1996         }
1997       }
1998     }
1999   }
2000 }
2001
2002 int mchecksum()
2003 {
2004   //if(!tracedebug) return 0;
2005   int i;
2006   int sum=0;
2007   for(i=0;i<2097152;i++) {
2008     unsigned int temp=sum;
2009     sum<<=1;
2010     sum|=(~temp)>>31;
2011     sum^=((u_int *)rdram)[i];
2012   }
2013   return sum;
2014 }
2015 int rchecksum()
2016 {
2017   int i;
2018   int sum=0;
2019   for(i=0;i<64;i++)
2020     sum^=((u_int *)reg)[i];
2021   return sum;
2022 }
2023 void rlist()
2024 {
2025   int i;
2026   printf("TRACE: ");
2027   for(i=0;i<32;i++)
2028     printf("r%d:%8x%8x ",i,((int *)(reg+i))[1],((int *)(reg+i))[0]);
2029   printf("\n");
2030 #ifndef DISABLE_COP1
2031   printf("TRACE: ");
2032   for(i=0;i<32;i++)
2033     printf("f%d:%8x%8x ",i,((int*)reg_cop1_simple[i])[1],*((int*)reg_cop1_simple[i]));
2034   printf("\n");
2035 #endif
2036 }
2037
2038 void enabletrace()
2039 {
2040   tracedebug=1;
2041 }
2042
2043 void memdebug(int i)
2044 {
2045   //printf("TRACE: count=%d next=%d (checksum %x) lo=%8x%8x\n",Count,next_interupt,mchecksum(),(int)(reg[LOREG]>>32),(int)reg[LOREG]);
2046   //printf("TRACE: count=%d next=%d (rchecksum %x)\n",Count,next_interupt,rchecksum());
2047   //rlist();
2048   //if(tracedebug) {
2049   //if(Count>=-2084597794) {
2050   if((signed int)Count>=-2084597794&&(signed int)Count<0) {
2051   //if(0) {
2052     printf("TRACE: count=%d next=%d (checksum %x)\n",Count,next_interupt,mchecksum());
2053     //printf("TRACE: count=%d next=%d (checksum %x) Status=%x\n",Count,next_interupt,mchecksum(),Status);
2054     //printf("TRACE: count=%d next=%d (checksum %x) hi=%8x%8x\n",Count,next_interupt,mchecksum(),(int)(reg[HIREG]>>32),(int)reg[HIREG]);
2055     rlist();
2056     #ifdef __i386__
2057     printf("TRACE: %x\n",(&i)[-1]);
2058     #endif
2059     #ifdef __arm__
2060     int j;
2061     printf("TRACE: %x \n",(&j)[10]);
2062     printf("TRACE: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x\n",(&j)[1],(&j)[2],(&j)[3],(&j)[4],(&j)[5],(&j)[6],(&j)[7],(&j)[8],(&j)[9],(&j)[10],(&j)[11],(&j)[12],(&j)[13],(&j)[14],(&j)[15],(&j)[16],(&j)[17],(&j)[18],(&j)[19],(&j)[20]);
2063     #endif
2064     //fflush(stdout);
2065   }
2066   //printf("TRACE: %x\n",(&i)[-1]);
2067 }
2068
2069 void tlb_debug(u_int cause, u_int addr, u_int iaddr)
2070 {
2071   printf("TLB Exception: instruction=%x addr=%x cause=%x\n",iaddr, addr, cause);
2072 }
2073
2074 void alu_assemble(int i,struct regstat *i_regs)
2075 {
2076   if(opcode2[i]>=0x20&&opcode2[i]<=0x23) { // ADD/ADDU/SUB/SUBU
2077     if(rt1[i]) {
2078       signed char s1,s2,t;
2079       t=get_reg(i_regs->regmap,rt1[i]);
2080       if(t>=0) {
2081         s1=get_reg(i_regs->regmap,rs1[i]);
2082         s2=get_reg(i_regs->regmap,rs2[i]);
2083         if(rs1[i]&&rs2[i]) {
2084           assert(s1>=0);
2085           assert(s2>=0);
2086           if(opcode2[i]&2) emit_sub(s1,s2,t);
2087           else emit_add(s1,s2,t);
2088         }
2089         else if(rs1[i]) {
2090           if(s1>=0) emit_mov(s1,t);
2091           else emit_loadreg(rs1[i],t);
2092         }
2093         else if(rs2[i]) {
2094           if(s2>=0) {
2095             if(opcode2[i]&2) emit_neg(s2,t);
2096             else emit_mov(s2,t);
2097           }
2098           else {
2099             emit_loadreg(rs2[i],t);
2100             if(opcode2[i]&2) emit_neg(t,t);
2101           }
2102         }
2103         else emit_zeroreg(t);
2104       }
2105     }
2106   }
2107   if(opcode2[i]>=0x2c&&opcode2[i]<=0x2f) { // DADD/DADDU/DSUB/DSUBU
2108     if(rt1[i]) {
2109       signed char s1l,s2l,s1h,s2h,tl,th;
2110       tl=get_reg(i_regs->regmap,rt1[i]);
2111       th=get_reg(i_regs->regmap,rt1[i]|64);
2112       if(tl>=0) {
2113         s1l=get_reg(i_regs->regmap,rs1[i]);
2114         s2l=get_reg(i_regs->regmap,rs2[i]);
2115         s1h=get_reg(i_regs->regmap,rs1[i]|64);
2116         s2h=get_reg(i_regs->regmap,rs2[i]|64);
2117         if(rs1[i]&&rs2[i]) {
2118           assert(s1l>=0);
2119           assert(s2l>=0);
2120           if(opcode2[i]&2) emit_subs(s1l,s2l,tl);
2121           else emit_adds(s1l,s2l,tl);
2122           if(th>=0) {
2123             #ifdef INVERTED_CARRY
2124             if(opcode2[i]&2) {if(s1h!=th) emit_mov(s1h,th);emit_sbb(th,s2h);}
2125             #else
2126             if(opcode2[i]&2) emit_sbc(s1h,s2h,th);
2127             #endif
2128             else emit_add(s1h,s2h,th);
2129           }
2130         }
2131         else if(rs1[i]) {
2132           if(s1l>=0) emit_mov(s1l,tl);
2133           else emit_loadreg(rs1[i],tl);
2134           if(th>=0) {
2135             if(s1h>=0) emit_mov(s1h,th);
2136             else emit_loadreg(rs1[i]|64,th);
2137           }
2138         }
2139         else if(rs2[i]) {
2140           if(s2l>=0) {
2141             if(opcode2[i]&2) emit_negs(s2l,tl);
2142             else emit_mov(s2l,tl);
2143           }
2144           else {
2145             emit_loadreg(rs2[i],tl);
2146             if(opcode2[i]&2) emit_negs(tl,tl);
2147           }
2148           if(th>=0) {
2149             #ifdef INVERTED_CARRY
2150             if(s2h>=0) emit_mov(s2h,th);
2151             else emit_loadreg(rs2[i]|64,th);
2152             if(opcode2[i]&2) {
2153               emit_adcimm(-1,th); // x86 has inverted carry flag
2154               emit_not(th,th);
2155             }
2156             #else
2157             if(opcode2[i]&2) {
2158               if(s2h>=0) emit_rscimm(s2h,0,th);
2159               else {
2160                 emit_loadreg(rs2[i]|64,th);
2161                 emit_rscimm(th,0,th);
2162               }
2163             }else{
2164               if(s2h>=0) emit_mov(s2h,th);
2165               else emit_loadreg(rs2[i]|64,th);
2166             }
2167             #endif
2168           }
2169         }
2170         else {
2171           emit_zeroreg(tl);
2172           if(th>=0) emit_zeroreg(th);
2173         }
2174       }
2175     }
2176   }
2177   if(opcode2[i]==0x2a||opcode2[i]==0x2b) { // SLT/SLTU
2178     if(rt1[i]) {
2179       signed char s1l,s1h,s2l,s2h,t;
2180       if(!((i_regs->was32>>rs1[i])&(i_regs->was32>>rs2[i])&1))
2181       {
2182         t=get_reg(i_regs->regmap,rt1[i]);
2183         //assert(t>=0);
2184         if(t>=0) {
2185           s1l=get_reg(i_regs->regmap,rs1[i]);
2186           s1h=get_reg(i_regs->regmap,rs1[i]|64);
2187           s2l=get_reg(i_regs->regmap,rs2[i]);
2188           s2h=get_reg(i_regs->regmap,rs2[i]|64);
2189           if(rs2[i]==0) // rx<r0
2190           {
2191             assert(s1h>=0);
2192             if(opcode2[i]==0x2a) // SLT
2193               emit_shrimm(s1h,31,t);
2194             else // SLTU (unsigned can not be less than zero)
2195               emit_zeroreg(t);
2196           }
2197           else if(rs1[i]==0) // r0<rx
2198           {
2199             assert(s2h>=0);
2200             if(opcode2[i]==0x2a) // SLT
2201               emit_set_gz64_32(s2h,s2l,t);
2202             else // SLTU (set if not zero)
2203               emit_set_nz64_32(s2h,s2l,t);
2204           }
2205           else {
2206             assert(s1l>=0);assert(s1h>=0);
2207             assert(s2l>=0);assert(s2h>=0);
2208             if(opcode2[i]==0x2a) // SLT
2209               emit_set_if_less64_32(s1h,s1l,s2h,s2l,t);
2210             else // SLTU
2211               emit_set_if_carry64_32(s1h,s1l,s2h,s2l,t);
2212           }
2213         }
2214       } else {
2215         t=get_reg(i_regs->regmap,rt1[i]);
2216         //assert(t>=0);
2217         if(t>=0) {
2218           s1l=get_reg(i_regs->regmap,rs1[i]);
2219           s2l=get_reg(i_regs->regmap,rs2[i]);
2220           if(rs2[i]==0) // rx<r0
2221           {
2222             assert(s1l>=0);
2223             if(opcode2[i]==0x2a) // SLT
2224               emit_shrimm(s1l,31,t);
2225             else // SLTU (unsigned can not be less than zero)
2226               emit_zeroreg(t);
2227           }
2228           else if(rs1[i]==0) // r0<rx
2229           {
2230             assert(s2l>=0);
2231             if(opcode2[i]==0x2a) // SLT
2232               emit_set_gz32(s2l,t);
2233             else // SLTU (set if not zero)
2234               emit_set_nz32(s2l,t);
2235           }
2236           else{
2237             assert(s1l>=0);assert(s2l>=0);
2238             if(opcode2[i]==0x2a) // SLT
2239               emit_set_if_less32(s1l,s2l,t);
2240             else // SLTU
2241               emit_set_if_carry32(s1l,s2l,t);
2242           }
2243         }
2244       }
2245     }
2246   }
2247   if(opcode2[i]>=0x24&&opcode2[i]<=0x27) { // AND/OR/XOR/NOR
2248     if(rt1[i]) {
2249       signed char s1l,s1h,s2l,s2h,th,tl;
2250       tl=get_reg(i_regs->regmap,rt1[i]);
2251       th=get_reg(i_regs->regmap,rt1[i]|64);
2252       if(!((i_regs->was32>>rs1[i])&(i_regs->was32>>rs2[i])&1)&&th>=0)
2253       {
2254         assert(tl>=0);
2255         if(tl>=0) {
2256           s1l=get_reg(i_regs->regmap,rs1[i]);
2257           s1h=get_reg(i_regs->regmap,rs1[i]|64);
2258           s2l=get_reg(i_regs->regmap,rs2[i]);
2259           s2h=get_reg(i_regs->regmap,rs2[i]|64);
2260           if(rs1[i]&&rs2[i]) {
2261             assert(s1l>=0);assert(s1h>=0);
2262             assert(s2l>=0);assert(s2h>=0);
2263             if(opcode2[i]==0x24) { // AND
2264               emit_and(s1l,s2l,tl);
2265               emit_and(s1h,s2h,th);
2266             } else
2267             if(opcode2[i]==0x25) { // OR
2268               emit_or(s1l,s2l,tl);
2269               emit_or(s1h,s2h,th);
2270             } else
2271             if(opcode2[i]==0x26) { // XOR
2272               emit_xor(s1l,s2l,tl);
2273               emit_xor(s1h,s2h,th);
2274             } else
2275             if(opcode2[i]==0x27) { // NOR
2276               emit_or(s1l,s2l,tl);
2277               emit_or(s1h,s2h,th);
2278               emit_not(tl,tl);
2279               emit_not(th,th);
2280             }
2281           }
2282           else
2283           {
2284             if(opcode2[i]==0x24) { // AND
2285               emit_zeroreg(tl);
2286               emit_zeroreg(th);
2287             } else
2288             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2289               if(rs1[i]){
2290                 if(s1l>=0) emit_mov(s1l,tl);
2291                 else emit_loadreg(rs1[i],tl);
2292                 if(s1h>=0) emit_mov(s1h,th);
2293                 else emit_loadreg(rs1[i]|64,th);
2294               }
2295               else
2296               if(rs2[i]){
2297                 if(s2l>=0) emit_mov(s2l,tl);
2298                 else emit_loadreg(rs2[i],tl);
2299                 if(s2h>=0) emit_mov(s2h,th);
2300                 else emit_loadreg(rs2[i]|64,th);
2301               }
2302               else{
2303                 emit_zeroreg(tl);
2304                 emit_zeroreg(th);
2305               }
2306             } else
2307             if(opcode2[i]==0x27) { // NOR
2308               if(rs1[i]){
2309                 if(s1l>=0) emit_not(s1l,tl);
2310                 else{
2311                   emit_loadreg(rs1[i],tl);
2312                   emit_not(tl,tl);
2313                 }
2314                 if(s1h>=0) emit_not(s1h,th);
2315                 else{
2316                   emit_loadreg(rs1[i]|64,th);
2317                   emit_not(th,th);
2318                 }
2319               }
2320               else
2321               if(rs2[i]){
2322                 if(s2l>=0) emit_not(s2l,tl);
2323                 else{
2324                   emit_loadreg(rs2[i],tl);
2325                   emit_not(tl,tl);
2326                 }
2327                 if(s2h>=0) emit_not(s2h,th);
2328                 else{
2329                   emit_loadreg(rs2[i]|64,th);
2330                   emit_not(th,th);
2331                 }
2332               }
2333               else {
2334                 emit_movimm(-1,tl);
2335                 emit_movimm(-1,th);
2336               }
2337             }
2338           }
2339         }
2340       }
2341       else
2342       {
2343         // 32 bit
2344         if(tl>=0) {
2345           s1l=get_reg(i_regs->regmap,rs1[i]);
2346           s2l=get_reg(i_regs->regmap,rs2[i]);
2347           if(rs1[i]&&rs2[i]) {
2348             assert(s1l>=0);
2349             assert(s2l>=0);
2350             if(opcode2[i]==0x24) { // AND
2351               emit_and(s1l,s2l,tl);
2352             } else
2353             if(opcode2[i]==0x25) { // OR
2354               emit_or(s1l,s2l,tl);
2355             } else
2356             if(opcode2[i]==0x26) { // XOR
2357               emit_xor(s1l,s2l,tl);
2358             } else
2359             if(opcode2[i]==0x27) { // NOR
2360               emit_or(s1l,s2l,tl);
2361               emit_not(tl,tl);
2362             }
2363           }
2364           else
2365           {
2366             if(opcode2[i]==0x24) { // AND
2367               emit_zeroreg(tl);
2368             } else
2369             if(opcode2[i]==0x25||opcode2[i]==0x26) { // OR/XOR
2370               if(rs1[i]){
2371                 if(s1l>=0) emit_mov(s1l,tl);
2372                 else emit_loadreg(rs1[i],tl); // CHECK: regmap_entry?
2373               }
2374               else
2375               if(rs2[i]){
2376                 if(s2l>=0) emit_mov(s2l,tl);
2377                 else emit_loadreg(rs2[i],tl); // CHECK: regmap_entry?
2378               }
2379               else emit_zeroreg(tl);
2380             } else
2381             if(opcode2[i]==0x27) { // NOR
2382               if(rs1[i]){
2383                 if(s1l>=0) emit_not(s1l,tl);
2384                 else {
2385                   emit_loadreg(rs1[i],tl);
2386                   emit_not(tl,tl);
2387                 }
2388               }
2389               else
2390               if(rs2[i]){
2391                 if(s2l>=0) emit_not(s2l,tl);
2392                 else {
2393                   emit_loadreg(rs2[i],tl);
2394                   emit_not(tl,tl);
2395                 }
2396               }
2397               else emit_movimm(-1,tl);
2398             }
2399           }
2400         }
2401       }
2402     }
2403   }
2404 }
2405
2406 void imm16_assemble(int i,struct regstat *i_regs)
2407 {
2408   if (opcode[i]==0x0f) { // LUI
2409     if(rt1[i]) {
2410       signed char t;
2411       t=get_reg(i_regs->regmap,rt1[i]);
2412       //assert(t>=0);
2413       if(t>=0) {
2414         if(!((i_regs->isconst>>t)&1))
2415           emit_movimm(imm[i]<<16,t);
2416       }
2417     }
2418   }
2419   if(opcode[i]==0x08||opcode[i]==0x09) { // ADDI/ADDIU
2420     if(rt1[i]) {
2421       signed char s,t;
2422       t=get_reg(i_regs->regmap,rt1[i]);
2423       s=get_reg(i_regs->regmap,rs1[i]);
2424       if(rs1[i]) {
2425         //assert(t>=0);
2426         //assert(s>=0);
2427         if(t>=0) {
2428           if(!((i_regs->isconst>>t)&1)) {
2429             if(s<0) {
2430               if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2431               emit_addimm(t,imm[i],t);
2432             }else{
2433               if(!((i_regs->wasconst>>s)&1))
2434                 emit_addimm(s,imm[i],t);
2435               else
2436                 emit_movimm(constmap[i][s]+imm[i],t);
2437             }
2438           }
2439         }
2440       } else {
2441         if(t>=0) {
2442           if(!((i_regs->isconst>>t)&1))
2443             emit_movimm(imm[i],t);
2444         }
2445       }
2446     }
2447   }
2448   if(opcode[i]==0x18||opcode[i]==0x19) { // DADDI/DADDIU
2449     if(rt1[i]) {
2450       signed char sh,sl,th,tl;
2451       th=get_reg(i_regs->regmap,rt1[i]|64);
2452       tl=get_reg(i_regs->regmap,rt1[i]);
2453       sh=get_reg(i_regs->regmap,rs1[i]|64);
2454       sl=get_reg(i_regs->regmap,rs1[i]);
2455       if(tl>=0) {
2456         if(rs1[i]) {
2457           assert(sh>=0);
2458           assert(sl>=0);
2459           if(th>=0) {
2460             emit_addimm64_32(sh,sl,imm[i],th,tl);
2461           }
2462           else {
2463             emit_addimm(sl,imm[i],tl);
2464           }
2465         } else {
2466           emit_movimm(imm[i],tl);
2467           if(th>=0) emit_movimm(((signed int)imm[i])>>31,th);
2468         }
2469       }
2470     }
2471   }
2472   else if(opcode[i]==0x0a||opcode[i]==0x0b) { // SLTI/SLTIU
2473     if(rt1[i]) {
2474       //assert(rs1[i]!=0); // r0 might be valid, but it's probably a bug
2475       signed char sh,sl,t;
2476       t=get_reg(i_regs->regmap,rt1[i]);
2477       sh=get_reg(i_regs->regmap,rs1[i]|64);
2478       sl=get_reg(i_regs->regmap,rs1[i]);
2479       //assert(t>=0);
2480       if(t>=0) {
2481         if(rs1[i]>0) {
2482           if(sh<0) assert((i_regs->was32>>rs1[i])&1);
2483           if(sh<0||((i_regs->was32>>rs1[i])&1)) {
2484             if(opcode[i]==0x0a) { // SLTI
2485               if(sl<0) {
2486                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2487                 emit_slti32(t,imm[i],t);
2488               }else{
2489                 emit_slti32(sl,imm[i],t);
2490               }
2491             }
2492             else { // SLTIU
2493               if(sl<0) {
2494                 if(i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2495                 emit_sltiu32(t,imm[i],t);
2496               }else{
2497                 emit_sltiu32(sl,imm[i],t);
2498               }
2499             }
2500           }else{ // 64-bit
2501             assert(sl>=0);
2502             if(opcode[i]==0x0a) // SLTI
2503               emit_slti64_32(sh,sl,imm[i],t);
2504             else // SLTIU
2505               emit_sltiu64_32(sh,sl,imm[i],t);
2506           }
2507         }else{
2508           // SLTI(U) with r0 is just stupid,
2509           // nonetheless examples can be found
2510           if(opcode[i]==0x0a) // SLTI
2511             if(0<imm[i]) emit_movimm(1,t);
2512             else emit_zeroreg(t);
2513           else // SLTIU
2514           {
2515             if(imm[i]) emit_movimm(1,t);
2516             else emit_zeroreg(t);
2517           }
2518         }
2519       }
2520     }
2521   }
2522   else if(opcode[i]>=0x0c&&opcode[i]<=0x0e) { // ANDI/ORI/XORI
2523     if(rt1[i]) {
2524       signed char sh,sl,th,tl;
2525       th=get_reg(i_regs->regmap,rt1[i]|64);
2526       tl=get_reg(i_regs->regmap,rt1[i]);
2527       sh=get_reg(i_regs->regmap,rs1[i]|64);
2528       sl=get_reg(i_regs->regmap,rs1[i]);
2529       if(tl>=0 && !((i_regs->isconst>>tl)&1)) {
2530         if(opcode[i]==0x0c) //ANDI
2531         {
2532           if(rs1[i]) {
2533             if(sl<0) {
2534               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2535               emit_andimm(tl,imm[i],tl);
2536             }else{
2537               if(!((i_regs->wasconst>>sl)&1))
2538                 emit_andimm(sl,imm[i],tl);
2539               else
2540                 emit_movimm(constmap[i][sl]&imm[i],tl);
2541             }
2542           }
2543           else
2544             emit_zeroreg(tl);
2545           if(th>=0) emit_zeroreg(th);
2546         }
2547         else
2548         {
2549           if(rs1[i]) {
2550             if(sl<0) {
2551               if(i_regs->regmap_entry[tl]!=rs1[i]) emit_loadreg(rs1[i],tl);
2552             }
2553             if(th>=0) {
2554               if(sh<0) {
2555                 emit_loadreg(rs1[i]|64,th);
2556               }else{
2557                 emit_mov(sh,th);
2558               }
2559             }
2560             if(opcode[i]==0x0d) //ORI
2561             if(sl<0) {
2562               emit_orimm(tl,imm[i],tl);
2563             }else{
2564               if(!((i_regs->wasconst>>sl)&1))
2565                 emit_orimm(sl,imm[i],tl);
2566               else
2567                 emit_movimm(constmap[i][sl]|imm[i],tl);
2568             }
2569             if(opcode[i]==0x0e) //XORI
2570             if(sl<0) {
2571               emit_xorimm(tl,imm[i],tl);
2572             }else{
2573               if(!((i_regs->wasconst>>sl)&1))
2574                 emit_xorimm(sl,imm[i],tl);
2575               else
2576                 emit_movimm(constmap[i][sl]^imm[i],tl);
2577             }
2578           }
2579           else {
2580             emit_movimm(imm[i],tl);
2581             if(th>=0) emit_zeroreg(th);
2582           }
2583         }
2584       }
2585     }
2586   }
2587 }
2588
2589 void shiftimm_assemble(int i,struct regstat *i_regs)
2590 {
2591   if(opcode2[i]<=0x3) // SLL/SRL/SRA
2592   {
2593     if(rt1[i]) {
2594       signed char s,t;
2595       t=get_reg(i_regs->regmap,rt1[i]);
2596       s=get_reg(i_regs->regmap,rs1[i]);
2597       //assert(t>=0);
2598       if(t>=0){
2599         if(rs1[i]==0)
2600         {
2601           emit_zeroreg(t);
2602         }
2603         else
2604         {
2605           if(s<0&&i_regs->regmap_entry[t]!=rs1[i]) emit_loadreg(rs1[i],t);
2606           if(imm[i]) {
2607             if(opcode2[i]==0) // SLL
2608             {
2609               emit_shlimm(s<0?t:s,imm[i],t);
2610             }
2611             if(opcode2[i]==2) // SRL
2612             {
2613               emit_shrimm(s<0?t:s,imm[i],t);
2614             }
2615             if(opcode2[i]==3) // SRA
2616             {
2617               emit_sarimm(s<0?t:s,imm[i],t);
2618             }
2619           }else{
2620             // Shift by zero
2621             if(s>=0 && s!=t) emit_mov(s,t);
2622           }
2623         }
2624       }
2625       //emit_storereg(rt1[i],t); //DEBUG
2626     }
2627   }
2628   if(opcode2[i]>=0x38&&opcode2[i]<=0x3b) // DSLL/DSRL/DSRA
2629   {
2630     if(rt1[i]) {
2631       signed char sh,sl,th,tl;
2632       th=get_reg(i_regs->regmap,rt1[i]|64);
2633       tl=get_reg(i_regs->regmap,rt1[i]);
2634       sh=get_reg(i_regs->regmap,rs1[i]|64);
2635       sl=get_reg(i_regs->regmap,rs1[i]);
2636       if(tl>=0) {
2637         if(rs1[i]==0)
2638         {
2639           emit_zeroreg(tl);
2640           if(th>=0) emit_zeroreg(th);
2641         }
2642         else
2643         {
2644           assert(sl>=0);
2645           assert(sh>=0);
2646           if(imm[i]) {
2647             if(opcode2[i]==0x38) // DSLL
2648             {
2649               if(th>=0) emit_shldimm(sh,sl,imm[i],th);
2650               emit_shlimm(sl,imm[i],tl);
2651             }
2652             if(opcode2[i]==0x3a) // DSRL
2653             {
2654               emit_shrdimm(sl,sh,imm[i],tl);
2655               if(th>=0) emit_shrimm(sh,imm[i],th);
2656             }
2657             if(opcode2[i]==0x3b) // DSRA
2658             {
2659               emit_shrdimm(sl,sh,imm[i],tl);
2660               if(th>=0) emit_sarimm(sh,imm[i],th);
2661             }
2662           }else{
2663             // Shift by zero
2664             if(sl!=tl) emit_mov(sl,tl);
2665             if(th>=0&&sh!=th) emit_mov(sh,th);
2666           }
2667         }
2668       }
2669     }
2670   }
2671   if(opcode2[i]==0x3c) // DSLL32
2672   {
2673     if(rt1[i]) {
2674       signed char sl,tl,th;
2675       tl=get_reg(i_regs->regmap,rt1[i]);
2676       th=get_reg(i_regs->regmap,rt1[i]|64);
2677       sl=get_reg(i_regs->regmap,rs1[i]);
2678       if(th>=0||tl>=0){
2679         assert(tl>=0);
2680         assert(th>=0);
2681         assert(sl>=0);
2682         emit_mov(sl,th);
2683         emit_zeroreg(tl);
2684         if(imm[i]>32)
2685         {
2686           emit_shlimm(th,imm[i]&31,th);
2687         }
2688       }
2689     }
2690   }
2691   if(opcode2[i]==0x3e) // DSRL32
2692   {
2693     if(rt1[i]) {
2694       signed char sh,tl,th;
2695       tl=get_reg(i_regs->regmap,rt1[i]);
2696       th=get_reg(i_regs->regmap,rt1[i]|64);
2697       sh=get_reg(i_regs->regmap,rs1[i]|64);
2698       if(tl>=0){
2699         assert(sh>=0);
2700         emit_mov(sh,tl);
2701         if(th>=0) emit_zeroreg(th);
2702         if(imm[i]>32)
2703         {
2704           emit_shrimm(tl,imm[i]&31,tl);
2705         }
2706       }
2707     }
2708   }
2709   if(opcode2[i]==0x3f) // DSRA32
2710   {
2711     if(rt1[i]) {
2712       signed char sh,tl;
2713       tl=get_reg(i_regs->regmap,rt1[i]);
2714       sh=get_reg(i_regs->regmap,rs1[i]|64);
2715       if(tl>=0){
2716         assert(sh>=0);
2717         emit_mov(sh,tl);
2718         if(imm[i]>32)
2719         {
2720           emit_sarimm(tl,imm[i]&31,tl);
2721         }
2722       }
2723     }
2724   }
2725 }
2726
2727 #ifndef shift_assemble
2728 void shift_assemble(int i,struct regstat *i_regs)
2729 {
2730   printf("Need shift_assemble for this architecture.\n");
2731   exit(1);
2732 }
2733 #endif
2734
2735 void load_assemble(int i,struct regstat *i_regs)
2736 {
2737   int s,th,tl,addr,map=-1;
2738   int offset;
2739   int jaddr=0;
2740   int memtarget=0,c=0;
2741   u_int hr,reglist=0;
2742   th=get_reg(i_regs->regmap,rt1[i]|64);
2743   tl=get_reg(i_regs->regmap,rt1[i]);
2744   s=get_reg(i_regs->regmap,rs1[i]);
2745   offset=imm[i];
2746   for(hr=0;hr<HOST_REGS;hr++) {
2747     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
2748   }
2749   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
2750   if(s>=0) {
2751     c=(i_regs->wasconst>>s)&1;
2752     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
2753     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
2754   }
2755   //printf("load_assemble: c=%d\n",c);
2756   //if(c) printf("load_assemble: const=%x\n",(int)constmap[i][s]+offset);
2757   // FIXME: Even if the load is a NOP, we should check for pagefaults...
2758 #ifdef PCSX
2759   if(tl<0&&(!c||(((u_int)constmap[i][s]+offset)>>16)==0x1f80)
2760     ||rt1[i]==0) {
2761       // could be FIFO, must perform the read
2762       // ||dummy read
2763       assem_debug("(forced read)\n");
2764       tl=get_reg(i_regs->regmap,-1);
2765       assert(tl>=0);
2766   }
2767 #endif
2768   if(offset||s<0||c) addr=tl;
2769   else addr=s;
2770   if(tl>=0) {
2771     //assert(tl>=0);
2772     //assert(rt1[i]);
2773     reglist&=~(1<<tl);
2774     if(th>=0) reglist&=~(1<<th);
2775     if(!using_tlb) {
2776       if(!c) {
2777 //#define R29_HACK 1
2778         #ifdef R29_HACK
2779         // Strmnnrmn's speed hack
2780         if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
2781         #endif
2782         {
2783           emit_cmpimm(addr,RAM_SIZE);
2784           jaddr=(int)out;
2785           #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
2786           // Hint to branch predictor that the branch is unlikely to be taken
2787           if(rs1[i]>=28)
2788             emit_jno_unlikely(0);
2789           else
2790           #endif
2791           emit_jno(0);
2792         }
2793       }
2794     }else{ // using tlb
2795       int x=0;
2796       if (opcode[i]==0x20||opcode[i]==0x24) x=3; // LB/LBU
2797       if (opcode[i]==0x21||opcode[i]==0x25) x=2; // LH/LHU
2798       map=get_reg(i_regs->regmap,TLREG);
2799       assert(map>=0);
2800       map=do_tlb_r(addr,tl,map,x,-1,-1,c,constmap[i][s]+offset);
2801       do_tlb_r_branch(map,c,constmap[i][s]+offset,&jaddr);
2802     }
2803     if (opcode[i]==0x20) { // LB
2804       if(!c||memtarget) {
2805         #ifdef HOST_IMM_ADDR32
2806         if(c)
2807           emit_movsbl_tlb((constmap[i][s]+offset)^3,map,tl);
2808         else
2809         #endif
2810         {
2811           //emit_xorimm(addr,3,tl);
2812           //gen_tlb_addr_r(tl,map);
2813           //emit_movsbl_indexed((int)rdram-0x80000000,tl,tl);
2814           int x=0;
2815 #ifdef BIG_ENDIAN_MIPS
2816           if(!c) emit_xorimm(addr,3,tl);
2817           else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
2818 #else
2819           if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
2820           else if (tl!=addr) emit_mov(addr,tl);
2821 #endif
2822           emit_movsbl_indexed_tlb(x,tl,map,tl);
2823         }
2824         if(jaddr)
2825           add_stub(LOADB_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2826       }
2827       else
2828         inline_readstub(LOADB_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2829     }
2830     if (opcode[i]==0x21) { // LH
2831       if(!c||memtarget) {
2832         #ifdef HOST_IMM_ADDR32
2833         if(c)
2834           emit_movswl_tlb((constmap[i][s]+offset)^2,map,tl);
2835         else
2836         #endif
2837         {
2838           int x=0;
2839 #ifdef BIG_ENDIAN_MIPS
2840           if(!c) emit_xorimm(addr,2,tl);
2841           else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
2842 #else
2843           if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
2844           else if (tl!=addr) emit_mov(addr,tl);
2845 #endif
2846           //#ifdef
2847           //emit_movswl_indexed_tlb(x,tl,map,tl);
2848           //else
2849           if(map>=0) {
2850             gen_tlb_addr_r(tl,map);
2851             emit_movswl_indexed(x,tl,tl);
2852           }else
2853             emit_movswl_indexed((int)rdram-0x80000000+x,tl,tl);
2854         }
2855         if(jaddr)
2856           add_stub(LOADH_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2857       }
2858       else
2859         inline_readstub(LOADH_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2860     }
2861     if (opcode[i]==0x23) { // LW
2862       if(!c||memtarget) {
2863         //emit_readword_indexed((int)rdram-0x80000000,addr,tl);
2864         #ifdef HOST_IMM_ADDR32
2865         if(c)
2866           emit_readword_tlb(constmap[i][s]+offset,map,tl);
2867         else
2868         #endif
2869         emit_readword_indexed_tlb(0,addr,map,tl);
2870         if(jaddr)
2871           add_stub(LOADW_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2872       }
2873       else
2874         inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2875     }
2876     if (opcode[i]==0x24) { // LBU
2877       if(!c||memtarget) {
2878         #ifdef HOST_IMM_ADDR32
2879         if(c)
2880           emit_movzbl_tlb((constmap[i][s]+offset)^3,map,tl);
2881         else
2882         #endif
2883         {
2884           //emit_xorimm(addr,3,tl);
2885           //gen_tlb_addr_r(tl,map);
2886           //emit_movzbl_indexed((int)rdram-0x80000000,tl,tl);
2887           int x=0;
2888 #ifdef BIG_ENDIAN_MIPS
2889           if(!c) emit_xorimm(addr,3,tl);
2890           else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
2891 #else
2892           if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
2893           else if (tl!=addr) emit_mov(addr,tl);
2894 #endif
2895           emit_movzbl_indexed_tlb(x,tl,map,tl);
2896         }
2897         if(jaddr)
2898           add_stub(LOADBU_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2899       }
2900       else
2901         inline_readstub(LOADBU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2902     }
2903     if (opcode[i]==0x25) { // LHU
2904       if(!c||memtarget) {
2905         #ifdef HOST_IMM_ADDR32
2906         if(c)
2907           emit_movzwl_tlb((constmap[i][s]+offset)^2,map,tl);
2908         else
2909         #endif
2910         {
2911           int x=0;
2912 #ifdef BIG_ENDIAN_MIPS
2913           if(!c) emit_xorimm(addr,2,tl);
2914           else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
2915 #else
2916           if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
2917           else if (tl!=addr) emit_mov(addr,tl);
2918 #endif
2919           //#ifdef
2920           //emit_movzwl_indexed_tlb(x,tl,map,tl);
2921           //#else
2922           if(map>=0) {
2923             gen_tlb_addr_r(tl,map);
2924             emit_movzwl_indexed(x,tl,tl);
2925           }else
2926             emit_movzwl_indexed((int)rdram-0x80000000+x,tl,tl);
2927           if(jaddr)
2928             add_stub(LOADHU_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2929         }
2930       }
2931       else
2932         inline_readstub(LOADHU_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2933     }
2934     if (opcode[i]==0x27) { // LWU
2935       assert(th>=0);
2936       if(!c||memtarget) {
2937         //emit_readword_indexed((int)rdram-0x80000000,addr,tl);
2938         #ifdef HOST_IMM_ADDR32
2939         if(c)
2940           emit_readword_tlb(constmap[i][s]+offset,map,tl);
2941         else
2942         #endif
2943         emit_readword_indexed_tlb(0,addr,map,tl);
2944         if(jaddr)
2945           add_stub(LOADW_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2946       }
2947       else {
2948         inline_readstub(LOADW_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2949       }
2950       emit_zeroreg(th);
2951     }
2952     if (opcode[i]==0x37) { // LD
2953       if(!c||memtarget) {
2954         //gen_tlb_addr_r(tl,map);
2955         //if(th>=0) emit_readword_indexed((int)rdram-0x80000000,addr,th);
2956         //emit_readword_indexed((int)rdram-0x7FFFFFFC,addr,tl);
2957         #ifdef HOST_IMM_ADDR32
2958         if(c)
2959           emit_readdword_tlb(constmap[i][s]+offset,map,th,tl);
2960         else
2961         #endif
2962         emit_readdword_indexed_tlb(0,addr,map,th,tl);
2963         if(jaddr)
2964           add_stub(LOADD_STUB,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
2965       }
2966       else
2967         inline_readstub(LOADD_STUB,i,constmap[i][s]+offset,i_regs->regmap,rt1[i],ccadj[i],reglist);
2968     }
2969     //emit_storereg(rt1[i],tl); // DEBUG
2970   }
2971   //if(opcode[i]==0x23)
2972   //if(opcode[i]==0x24)
2973   //if(opcode[i]==0x23||opcode[i]==0x24)
2974   /*if(opcode[i]==0x21||opcode[i]==0x23||opcode[i]==0x24)
2975   {
2976     //emit_pusha();
2977     save_regs(0x100f);
2978         emit_readword((int)&last_count,ECX);
2979         #ifdef __i386__
2980         if(get_reg(i_regs->regmap,CCREG)<0)
2981           emit_loadreg(CCREG,HOST_CCREG);
2982         emit_add(HOST_CCREG,ECX,HOST_CCREG);
2983         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
2984         emit_writeword(HOST_CCREG,(int)&Count);
2985         #endif
2986         #ifdef __arm__
2987         if(get_reg(i_regs->regmap,CCREG)<0)
2988           emit_loadreg(CCREG,0);
2989         else
2990           emit_mov(HOST_CCREG,0);
2991         emit_add(0,ECX,0);
2992         emit_addimm(0,2*ccadj[i],0);
2993         emit_writeword(0,(int)&Count);
2994         #endif
2995     emit_call((int)memdebug);
2996     //emit_popa();
2997     restore_regs(0x100f);
2998   }/**/
2999 }
3000
3001 #ifndef loadlr_assemble
3002 void loadlr_assemble(int i,struct regstat *i_regs)
3003 {
3004   printf("Need loadlr_assemble for this architecture.\n");
3005   exit(1);
3006 }
3007 #endif
3008
3009 void store_assemble(int i,struct regstat *i_regs)
3010 {
3011   int s,th,tl,map=-1;
3012   int addr,temp;
3013   int offset;
3014   int jaddr=0,jaddr2,type;
3015   int memtarget=0,c=0;
3016   int agr=AGEN1+(i&1);
3017   u_int hr,reglist=0;
3018   th=get_reg(i_regs->regmap,rs2[i]|64);
3019   tl=get_reg(i_regs->regmap,rs2[i]);
3020   s=get_reg(i_regs->regmap,rs1[i]);
3021   temp=get_reg(i_regs->regmap,agr);
3022   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3023   offset=imm[i];
3024   if(s>=0) {
3025     c=(i_regs->wasconst>>s)&1;
3026     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
3027     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
3028   }
3029   assert(tl>=0);
3030   assert(temp>=0);
3031   for(hr=0;hr<HOST_REGS;hr++) {
3032     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3033   }
3034   if(i_regs->regmap[HOST_CCREG]==CCREG) reglist&=~(1<<HOST_CCREG);
3035   if(offset||s<0||c) addr=temp;
3036   else addr=s;
3037   if(!using_tlb) {
3038     if(!c) {
3039       #ifdef R29_HACK
3040       // Strmnnrmn's speed hack
3041       memtarget=1;
3042       if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
3043       #endif
3044       emit_cmpimm(addr,RAM_SIZE);
3045       #ifdef DESTRUCTIVE_SHIFT
3046       if(s==addr) emit_mov(s,temp);
3047       #endif
3048       #ifdef R29_HACK
3049       if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE)
3050       #endif
3051       {
3052         jaddr=(int)out;
3053         #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK
3054         // Hint to branch predictor that the branch is unlikely to be taken
3055         if(rs1[i]>=28)
3056           emit_jno_unlikely(0);
3057         else
3058         #endif
3059         emit_jno(0);
3060       }
3061     }
3062   }else{ // using tlb
3063     int x=0;
3064     if (opcode[i]==0x28) x=3; // SB
3065     if (opcode[i]==0x29) x=2; // SH
3066     map=get_reg(i_regs->regmap,TLREG);
3067     assert(map>=0);
3068     map=do_tlb_w(addr,temp,map,x,c,constmap[i][s]+offset);
3069     do_tlb_w_branch(map,c,constmap[i][s]+offset,&jaddr);
3070   }
3071
3072   if (opcode[i]==0x28) { // SB
3073     if(!c||memtarget) {
3074       int x=0;
3075 #ifdef BIG_ENDIAN_MIPS
3076       if(!c) emit_xorimm(addr,3,temp);
3077       else x=((constmap[i][s]+offset)^3)-(constmap[i][s]+offset);
3078 #else
3079       if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
3080       else if (addr!=temp) emit_mov(addr,temp);
3081 #endif
3082       //gen_tlb_addr_w(temp,map);
3083       //emit_writebyte_indexed(tl,(int)rdram-0x80000000,temp);
3084       emit_writebyte_indexed_tlb(tl,x,temp,map,temp);
3085     }
3086     type=STOREB_STUB;
3087   }
3088   if (opcode[i]==0x29) { // SH
3089     if(!c||memtarget) {
3090       int x=0;
3091 #ifdef BIG_ENDIAN_MIPS
3092       if(!c) emit_xorimm(addr,2,temp);
3093       else x=((constmap[i][s]+offset)^2)-(constmap[i][s]+offset);
3094 #else
3095       if(c) x=(constmap[i][s]+offset)-(constmap[i][s]+offset);
3096       else if (addr!=temp) emit_mov(addr,temp);
3097 #endif
3098       //#ifdef
3099       //emit_writehword_indexed_tlb(tl,x,temp,map,temp);
3100       //#else
3101       if(map>=0) {
3102         gen_tlb_addr_w(temp,map);
3103         emit_writehword_indexed(tl,x,temp);
3104       }else
3105         emit_writehword_indexed(tl,(int)rdram-0x80000000+x,temp);
3106     }
3107     type=STOREH_STUB;
3108   }
3109   if (opcode[i]==0x2B) { // SW
3110     if(!c||memtarget)
3111       //emit_writeword_indexed(tl,(int)rdram-0x80000000,addr);
3112       emit_writeword_indexed_tlb(tl,0,addr,map,temp);
3113     type=STOREW_STUB;
3114   }
3115   if (opcode[i]==0x3F) { // SD
3116     if(!c||memtarget) {
3117       if(rs2[i]) {
3118         assert(th>=0);
3119         //emit_writeword_indexed(th,(int)rdram-0x80000000,addr);
3120         //emit_writeword_indexed(tl,(int)rdram-0x7FFFFFFC,addr);
3121         emit_writedword_indexed_tlb(th,tl,0,addr,map,temp);
3122       }else{
3123         // Store zero
3124         //emit_writeword_indexed(tl,(int)rdram-0x80000000,temp);
3125         //emit_writeword_indexed(tl,(int)rdram-0x7FFFFFFC,temp);
3126         emit_writedword_indexed_tlb(tl,tl,0,addr,map,temp);
3127       }
3128     }
3129     type=STORED_STUB;
3130   }
3131   if(!using_tlb&&(!c||memtarget))
3132     // addr could be a temp, make sure it survives STORE*_STUB
3133     reglist|=1<<addr;
3134   if(jaddr) {
3135     add_stub(type,jaddr,(int)out,i,addr,(int)i_regs,ccadj[i],reglist);
3136   } else if(!memtarget) {
3137     inline_writestub(type,i,constmap[i][s]+offset,i_regs->regmap,rs2[i],ccadj[i],reglist);
3138   }
3139   if(!using_tlb) {
3140     if(!c||memtarget) {
3141       #ifdef DESTRUCTIVE_SHIFT
3142       // The x86 shift operation is 'destructive'; it overwrites the
3143       // source register, so we need to make a copy first and use that.
3144       addr=temp;
3145       #endif
3146       #if defined(HOST_IMM8)
3147       int ir=get_reg(i_regs->regmap,INVCP);
3148       assert(ir>=0);
3149       emit_cmpmem_indexedsr12_reg(ir,addr,1);
3150       #else
3151       emit_cmpmem_indexedsr12_imm((int)invalid_code,addr,1);
3152       #endif
3153       jaddr2=(int)out;
3154       emit_jne(0);
3155       add_stub(INVCODE_STUB,jaddr2,(int)out,reglist|(1<<HOST_CCREG),addr,0,0,0);
3156     }
3157   }
3158   //if(opcode[i]==0x2B || opcode[i]==0x3F)
3159   //if(opcode[i]==0x2B || opcode[i]==0x28)
3160   //if(opcode[i]==0x2B || opcode[i]==0x29)
3161   //if(opcode[i]==0x2B)
3162   /*if(opcode[i]==0x2B || opcode[i]==0x28 || opcode[i]==0x29 || opcode[i]==0x3F)
3163   {
3164     //emit_pusha();
3165     save_regs(0x100f);
3166         emit_readword((int)&last_count,ECX);
3167         #ifdef __i386__
3168         if(get_reg(i_regs->regmap,CCREG)<0)
3169           emit_loadreg(CCREG,HOST_CCREG);
3170         emit_add(HOST_CCREG,ECX,HOST_CCREG);
3171         emit_addimm(HOST_CCREG,2*ccadj[i],HOST_CCREG);
3172         emit_writeword(HOST_CCREG,(int)&Count);
3173         #endif
3174         #ifdef __arm__
3175         if(get_reg(i_regs->regmap,CCREG)<0)
3176           emit_loadreg(CCREG,0);
3177         else
3178           emit_mov(HOST_CCREG,0);
3179         emit_add(0,ECX,0);
3180         emit_addimm(0,2*ccadj[i],0);
3181         emit_writeword(0,(int)&Count);
3182         #endif
3183     emit_call((int)memdebug);
3184     //emit_popa();
3185     restore_regs(0x100f);
3186   }/**/
3187 }
3188
3189 void storelr_assemble(int i,struct regstat *i_regs)
3190 {
3191   int s,th,tl;
3192   int temp;
3193   int temp2;
3194   int offset;
3195   int jaddr=0,jaddr2;
3196   int case1,case2,case3;
3197   int done0,done1,done2;
3198   int memtarget,c=0;
3199   int agr=AGEN1+(i&1);
3200   u_int hr,reglist=0;
3201   th=get_reg(i_regs->regmap,rs2[i]|64);
3202   tl=get_reg(i_regs->regmap,rs2[i]);
3203   s=get_reg(i_regs->regmap,rs1[i]);
3204   temp=get_reg(i_regs->regmap,agr);
3205   if(temp<0) temp=get_reg(i_regs->regmap,-1);
3206   offset=imm[i];
3207   if(s>=0) {
3208     c=(i_regs->isconst>>s)&1;
3209     memtarget=((signed int)(constmap[i][s]+offset))<(signed int)0x80000000+RAM_SIZE;
3210     if(using_tlb&&((signed int)(constmap[i][s]+offset))>=(signed int)0xC0000000) memtarget=1;
3211   }
3212   assert(tl>=0);
3213   for(hr=0;hr<HOST_REGS;hr++) {
3214     if(i_regs->regmap[hr]>=0) reglist|=1<<hr;
3215   }
3216   if(tl>=0) {
3217     assert(temp>=0);
3218     if(!using_tlb) {
3219       if(!c) {
3220         emit_cmpimm(s<0||offset?temp:s,RAM_SIZE);
3221         if(!offset&&s!=temp) emit_mov(s,temp);
3222         jaddr=(int)out;
3223         emit_jno(0);
3224       }
3225       else
3226       {
3227         if(!memtarget||!rs1[i]) {
3228           jaddr=(int)out;
3229           emit_jmp(0);
3230         }
3231       }
3232       if((u_int)rdram!=0x80000000)
3233         emit_addimm_no_flags((u_int)rdram-(u_int)0x80000000,temp);
3234     }else{ // using tlb
3235       int map=get_reg(i_regs->regmap,TLREG);
3236       assert(map>=0);
3237       map=do_tlb_w(c||s<0||offset?temp:s,temp,map,0,c,constmap[i][s]+offset);
3238       if(!c&&!offset&&s>=0) emit_mov(s,temp);
3239       do_tlb_w_branch(map,c,constmap[i][s]+offset,&jaddr);
3240       if(!jaddr&&!memtarget) {
3241         jaddr=(int)out;
3242         emit_jmp(0);
3243       }
3244       gen_tlb_addr_w(temp,map);
3245     }
3246
3247     if (opcode[i]==0x2C||opcode[i]==0x2D) { // SDL/SDR
3248       temp2=get_reg(i_regs->regmap,FTEMP);
3249       if(!rs2[i]) temp2=th=tl;
3250     }
3251
3252 #ifndef BIG_ENDIAN_MIPS
3253     emit_xorimm(temp,3,temp);
3254 #endif
3255     emit_testimm(temp,2);
3256     case2=(int)out;
3257     emit_jne(0);
3258     emit_testimm(temp,1);
3259     case1=(int)out;
3260     emit_jne(0);
3261     // 0
3262     if (opcode[i]==0x2A) { // SWL
3263       emit_writeword_indexed(tl,0,temp);
3264     }
3265     if (opcode[i]==0x2E) { // SWR
3266       emit_writebyte_indexed(tl,3,temp);
3267     }
3268     if (opcode[i]==0x2C) { // SDL
3269       emit_writeword_indexed(th,0,temp);
3270       if(rs2[i]) emit_mov(tl,temp2);
3271     }
3272     if (opcode[i]==0x2D) { // SDR
3273       emit_writebyte_indexed(tl,3,temp);
3274       if(rs2[i]) emit_shldimm(th,tl,24,temp2);
3275     }
3276     done0=(int)out;
3277     emit_jmp(0);
3278     // 1
3279     set_jump_target(case1,(int)out);
3280     if (opcode[i]==0x2A) { // SWL
3281       // Write 3 msb into three least significant bytes
3282       if(rs2[i]) emit_rorimm(tl,8,tl);
3283       emit_writehword_indexed(tl,-1,temp);
3284       if(rs2[i]) emit_rorimm(tl,16,tl);
3285       emit_writebyte_indexed(tl,1,temp);
3286       if(rs2[i]) emit_rorimm(tl,8,tl);
3287     }
3288     if (opcode[i]==0x2E) { // SWR
3289       // Write two lsb into two most significant bytes
3290       emit_writehword_indexed(tl,1,temp);
3291     }
3292     if (opcode[i]==0x2C) { // SDL
3293       if(rs2[i]) emit_shrdimm(tl,th,8,temp2);
3294       // Write 3 msb into three least significant bytes
3295       if(rs2[i]) emit_rorimm(th,8,th);
3296       emit_writehword_indexed(th,-1,temp);
3297       if(rs2[i]) emit_rorimm(th,16,th);
3298       emit_writebyte_indexed(th,1,temp);
3299       if(rs2[i]) emit_rorimm(th,8,th);
3300     }
3301     if (opcode[i]==0x2D) { // SDR
3302       if(rs2[i]) emit_shldimm(th,tl,16,temp2);
3303       // Write two lsb into two most significant bytes
3304       emit_writehword_indexed(tl,1,temp);
3305     }
3306     done1=(int)out;
3307     emit_jmp(0);
3308     // 2
3309     set_jump_target(case2,(int)out);
3310     emit_testimm(temp,1);
3311     case3=(int)out;
3312     emit_jne(0);
3313     if (opcode[i]==0x2A) { // SWL
3314       // Write two msb into two least significant bytes
3315       if(rs2[i]) emit_rorimm(tl,16,tl);
3316       emit_writehword_indexed(tl,-2,temp);
3317       if(rs2[i]) emit_rorimm(tl,16,tl);
3318     }
3319     if (opcode[i]==0x2E) { // SWR
3320       // Write 3 lsb into three most significant bytes
3321       emit_writebyte_indexed(tl,-1,temp);
3322       if(rs2[i]) emit_rorimm(tl,8,tl);
3323       emit_writehword_indexed(tl,0,temp);
3324       if(rs2[i]) emit_rorimm(tl,24,tl);
3325     }
3326     if (opcode[i]==0x2C) { // SDL
3327       if(rs2[i]) emit_shrdimm(tl,th,16,temp2);
3328       // Write two msb into two least significant bytes
3329       if(rs2[i]) emit_rorimm(th,16,th);
3330       emit_writehword_indexed(th,-2,temp);
3331       if(rs2[i]) emit_rorimm(th,16,th);
3332     }
3333     if (opcode[i]==0x2D) { // SDR
3334       if(rs2[i]) emit_shldimm(th,tl,8,temp2);
3335       // Write 3 lsb into three most significant bytes
3336       emit_writebyte_indexed(tl,-1,temp);
3337       if(rs2[i]) emit_rorimm(tl,8,tl);
3338       emit_writehword_indexed(tl,0,temp);
3339       if(rs2[i]) emit_rorimm(tl,24,tl);
3340     }
3341     done2=(int)out;
3342     emit_jmp(0);
3343     // 3
3344     set_jump_target(case3,(int)out);
3345     if (opcode[i]==0x2A) { // SWL
3346       // Write msb into least significant byte
3347       if(rs2[i]) emit_rorimm(tl,24,tl);
3348       emit_writebyte_indexed(tl,-3,temp);
3349       if(rs2[i]) emit_rorimm(tl,8,tl);
3350     }
3351     if (opcode[i]==0x2E) { // SWR
3352       // Write entire word
3353       emit_writeword_indexed(tl,-3,temp);
3354     }
3355     if (opcode[i]==0x2C) { // SDL
3356       if(rs2[i]) emit_shrdimm(tl,th,24,temp2);
3357       // Write msb into least significant byte
3358       if(rs2[i]) emit_rorimm(th,24,th);
3359       emit_writebyte_indexed(th,-3,temp);
3360       if(rs2[i]) emit_rorimm(th,8,th);
3361     }
3362     if (opcode[i]==0x2D) { // SDR
3363       if(rs2[i]) emit_mov(th,temp2);
3364       // Write entire word
3365       emit_writeword_indexed(tl,-3,temp);
3366     }
3367     set_jump_target(done0,(int)out);
3368     set_jump_target(done1,(int)out);
3369     set_jump_target(done2,(int)out);
3370     if (opcode[i]==0x2C) { // SDL
3371       emit_testimm(temp,4);