X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=libpcsxcore%2Fnew_dynarec%2Fassem_arm64.c;h=271bee5807e69d7663b20503ad0126c800fa5e99;hb=9b495f6ec3f28cf5ed1d41f6af16a9967fcf3e64;hp=27f9141d3a35eb198e1016d773edc61e20b5c8ab;hpb=3968e69e7fa8f9cb0d44ac79477d5929b9649271;p=pcsx_rearmed.git diff --git a/libpcsxcore/new_dynarec/assem_arm64.c b/libpcsxcore/new_dynarec/assem_arm64.c index 27f9141d..271bee58 100644 --- a/libpcsxcore/new_dynarec/assem_arm64.c +++ b/libpcsxcore/new_dynarec/assem_arm64.c @@ -23,25 +23,13 @@ #include "pcnt.h" #include "arm_features.h" -#if defined(BASE_ADDR_FIXED) -#elif defined(BASE_ADDR_DYNAMIC) -u_char *translation_cache; -#else -u_char translation_cache[1 << TARGET_SIZE_2] __attribute__((aligned(4096))); -#endif -static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)]; - -#define CALLER_SAVE_REGS 0x0007ffff - -#define unused __attribute__((unused)) - void do_memhandler_pre(); void do_memhandler_post(); /* Linker */ static void set_jump_target(void *addr, void *target) { - u_int *ptr = addr; + u_int *ptr = NDRC_WRITE_OFFSET(addr); intptr_t offset = (u_char *)target - (u_char *)addr; if ((*ptr&0xFC000000) == 0x14000000) { // b @@ -52,10 +40,10 @@ static void set_jump_target(void *addr, void *target) || (*ptr&0x7e000000) == 0x34000000) { // cbz/cbnz // Conditional branch are limited to +/- 1MB // block max size is 256k so branching beyond the +/- 1MB limit - // should only happen when jumping to an already compiled block (see add_link) + // should only happen when jumping to an already compiled block (see add_jump_out) // a workaround would be to do a trampoline jump via a stub at the end of the block assert(-1048576 <= offset && offset < 1048576); - *ptr=(*ptr&0xFF00000F)|(((offset>>2)&0x7ffff)<<5); + *ptr=(*ptr&0xFF00001F)|(((offset>>2)&0x7ffff)<<5); } else if((*ptr&0x9f000000)==0x10000000) { // adr // generated by do_miniht_insert @@ -76,6 +64,7 @@ static void *find_extjump_insn(void *stub) return ptr + offset / 4; } +#if 0 // find where external branch is liked to using addr of it's stub: // get address that the stub loads (dyna_linker arg1), // treat it as a pointer to branch insn, @@ -91,6 +80,7 @@ static void *get_pointer(void *stub) assert(0); return NULL; } +#endif // Allocate a specific ARM register. static void alloc_arm_reg(struct regstat *cur,int i,signed char reg,int hr) @@ -150,25 +140,10 @@ static unused const char *condname[16] = { static void output_w32(u_int word) { - *((u_int *)out) = word; + *((u_int *)NDRC_WRITE_OFFSET(out)) = word; out += 4; } -static void output_w64(uint64_t dword) -{ - *((uint64_t *)out) = dword; - out+=8; -} - -/* -static u_int rm_rd(u_int rm, u_int rd) -{ - assert(rm < 31); - assert(rd < 31); - return (rm << 16) | rd; -} -*/ - static u_int rn_rd(u_int rn, u_int rd) { assert(rn < 31); @@ -338,6 +313,7 @@ static void emit_adds64(u_int rs1, u_int rs2, u_int rt) assem_debug("adds %s,%s,%s\n",regname64[rt],regname64[rs1],regname64[rs2]); output_w32(0xab000000 | rm_rn_rd(rs2, rs1, rt)); } +#define emit_adds_ptr emit_adds64 static void emit_neg(u_int rs, u_int rt) { @@ -421,6 +397,27 @@ static void emit_movimm(u_int imm, u_int rt) } } +static void emit_movimm64(uint64_t imm, u_int rt) +{ + u_int shift, op, imm16, insns = 0; + for (shift = 0; shift < 4; shift++) { + imm16 = (imm >> shift * 16) & 0xffff; + if (!imm16) + continue; + op = insns ? 0xf2800000 : 0xd2800000; + assem_debug("mov%c %s,#%#x", insns ? 'k' : 'z', regname64[rt], imm16); + if (shift) + assem_debug(",lsl #%u", shift * 16); + assem_debug("\n"); + output_w32(op | (shift << 21) | imm16_rd(imm16, rt)); + insns++; + } + if (!insns) { + assem_debug("movz %s,#0\n", regname64[rt]); + output_w32(0xd2800000 | imm16_rd(0, rt)); + } +} + static void emit_readword(void *addr, u_int rt) { uintptr_t offset = (u_char *)addr - (u_char *)&dynarec_local; @@ -442,6 +439,7 @@ static void emit_readdword(void *addr, u_int rt) else abort(); } +#define emit_readptr emit_readdword static void emit_readshword(void *addr, u_int rt) { @@ -457,18 +455,21 @@ static void emit_readshword(void *addr, u_int rt) static void emit_loadreg(u_int r, u_int hr) { int is64 = 0; - assert(r < 64); if (r == 0) emit_zeroreg(hr); else { - void *addr = &psxRegs.GPR.r[r]; + void *addr; switch (r) { //case HIREG: addr = &hi; break; //case LOREG: addr = &lo; break; case CCREG: addr = &cycle_count; break; case CSREG: addr = &Status; break; case INVCP: addr = &invc_ptr; is64 = 1; break; - default: assert(r < 34); break; + case ROREG: addr = &ram_offset; is64 = 1; break; + default: + assert(r < 34); + addr = &psxRegs.GPR.r[r]; + break; } if (is64) emit_readdword(addr, hr); @@ -616,6 +617,10 @@ static void emit_addimm_s(u_int s, u_int is64, u_int rs, uintptr_t imm, u_int rt static void emit_addimm(u_int rs, uintptr_t imm, u_int rt) { + if (imm == 0) { + emit_mov(rs, rt); + return; + } emit_addimm_s(0, 0, rs, imm, rt); } @@ -629,11 +634,6 @@ static void emit_addimm_and_set_flags(int imm, u_int rt) emit_addimm_s(1, 0, rt, imm, rt); } -static void emit_addimm_no_flags(u_int imm,u_int rt) -{ - emit_addimm(rt,imm,rt); -} - static void emit_logicop_imm(u_int op, u_int rs, u_int imm, u_int rt) { const char *names[] = { "and", "orr", "eor", "ands" }; @@ -821,6 +821,12 @@ static void emit_cmovl_reg(u_int rs,u_int rt) output_w32(0x1a800000 | (COND_LT << 12) | rm_rn_rd(rt, rs, rt)); } +static void emit_cmovb_reg(u_int rs,u_int rt) +{ + assem_debug("csel %s,%s,%s,cc\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_CC << 12) | rm_rn_rd(rt, rs, rt)); +} + static void emit_cmovs_reg(u_int rs,u_int rt) { assem_debug("csel %s,%s,%s,mi\n",regname[rt],regname[rs],regname[rt]); @@ -855,6 +861,12 @@ static void emit_cmp(u_int rs,u_int rt) output_w32(0x6b000000 | rm_rn_rd(rt, rs, WZR)); } +static void emit_cmpcs(u_int rs,u_int rt) +{ + assem_debug("ccmp %s,%s,#0,cs\n",regname[rs],regname[rt]); + output_w32(0x7a400000 | (COND_CS << 12) | rm_rn_rd(rt, rs, 0)); +} + static void emit_set_gz32(u_int rs, u_int rt) { //assem_debug("set_gz32\n"); @@ -889,6 +901,12 @@ static void emit_set_if_carry32(u_int rs1, u_int rs2, u_int rt) emit_cmovb_imm(1,rt); } +static int can_jump_or_call(const void *a) +{ + intptr_t diff = (u_char *)a - out; + return (-134217728 <= diff && diff <= 134217727); +} + static void emit_call(const void *a) { intptr_t diff = (u_char *)a - out; @@ -972,9 +990,11 @@ static void emit_cb(u_int isnz, u_int is64, const void *a, u_int r) output_w32(0x34000000 | is64 | isnz | imm19_rt(offset, r)); } -static void emit_cbz(const void *a, u_int r) +static void *emit_cbz(u_int r, const void *a) { + void *ret = out; emit_cb(0, 0, a, r); + return ret; } static void emit_jmpreg(u_int r) @@ -1043,6 +1063,7 @@ static void emit_readdword_dualindexedx8(u_int rs1, u_int rs2, u_int rt) assem_debug("ldr %s, [%s,%s, uxtw #3]\n",regname64[rt],regname64[rs1],regname[rs2]); output_w32(0xf8605800 | rm_rn_rd(rs2, rs1, rt)); } +#define emit_readptr_dualindexedx_ptrlen emit_readdword_dualindexedx8 static void emit_ldrb_dualindexed(u_int rs1, u_int rs2, u_int rt) { @@ -1181,14 +1202,11 @@ static void emit_clz(u_int rs, u_int rt) } // special case for checking invalid_code -static void emit_cmpmem_indexedsr12_reg(u_int rbase, u_int r, u_int imm) +static void emit_ldrb_indexedsr12_reg(u_int rbase, u_int r, u_int rt) { - host_tempreg_acquire(); - emit_shrimm(r, 12, HOST_TEMPREG); - assem_debug("ldrb %s,[%s,%s,uxtw]\n",regname[HOST_TEMPREG],regname64[rbase],regname[HOST_TEMPREG]); - output_w32(0x38604800 | rm_rn_rd(HOST_TEMPREG, rbase, HOST_TEMPREG)); - emit_cmpimm(HOST_TEMPREG, imm); - host_tempreg_release(); + emit_shrimm(r, 12, rt); + assem_debug("ldrb %s,[%s,%s,uxtw]\n",regname[rt],regname64[rbase],regname[rt]); + output_w32(0x38604800 | rm_rn_rd(rt, rbase, rt)); } // special for loadlr_assemble, rs2 is destroyed @@ -1204,11 +1222,6 @@ static void emit_bic_lsr(u_int rs1,u_int rs2,u_int shift,u_int rt) emit_bic(rs1, rs2, rt); } -static void emit_loadlp_ofs(u_int ofs, u_int rt) -{ - output_w32(0x58000000 | imm19_rt(ofs, rt)); -} - static void emit_ldst(int is_st, int is64, u_int rt, u_int rn, u_int ofs) { u_int op = 0xb9000000; @@ -1285,7 +1298,7 @@ static void literal_pool_jumpover(int n) } // parsed by get_pointer, find_extjump_insn -static void emit_extjump2(u_char *addr, u_int target, void *linker) +static void emit_extjump(u_char *addr, u_int target) { assert(((addr[3]&0xfc)==0x14) || ((addr[3]&0xff)==0x54)); // b or b.cond @@ -1295,7 +1308,7 @@ static void emit_extjump2(u_char *addr, u_int target, void *linker) // addr is in the current recompiled block (max 256k) // offset shouldn't exceed +/-1MB emit_adr(addr, 1); - emit_jmp(linker); + emit_far_jump(dyna_linker); } static void check_extjump2(void *src) @@ -1330,7 +1343,18 @@ static int is_similar_value(u_int v1, u_int v2) || is_rotated_mask(v1 ^ v2); } -// trashes r2 +static void emit_movimm_from64(u_int rs_val, u_int rs, uintptr_t rt_val, u_int rt) +{ + if (rt_val < 0x100000000ull) { + emit_movimm_from(rs_val, rs, rt_val, rt); + return; + } + // just move the whole thing. At least on Linux all addresses + // seem to be 48bit, so 3 insns - not great not terrible + emit_movimm64(rt_val, rt); +} + +// trashes x2 static void pass_args64(u_int a0, u_int a1) { if(a0==1&&a1==0) { @@ -1376,10 +1400,10 @@ static void do_readstub(int n) u_int reglist = stubs[n].e; const signed char *i_regmap = i_regs->regmap; int rt; - if(itype[i]==C1LS||itype[i]==C2LS||itype[i]==LOADLR) { + if(dops[i].itype==C1LS||dops[i].itype==C2LS||dops[i].itype==LOADLR) { rt=get_reg(i_regmap,FTEMP); }else{ - rt=get_reg(i_regmap,rt1[i]); + rt=get_reg(i_regmap,dops[i].rt1); } assert(rs>=0); int r,temp=-1,temp2=HOST_TEMPREG,regs_saved=0; @@ -1391,7 +1415,7 @@ static void do_readstub(int n) break; } } - if(rt>=0&&rt1[i]!=0) + if(rt>=0&&dops[i].rt1!=0) reglist&=~(1<=0&&rt1[i]!=0)) { + if(dops[i].itype==C1LS||dops[i].itype==C2LS||(rt>=0&&dops[i].rt1!=0)) { switch(type) { case LOADB_STUB: emit_ldrsb_dualindexed(temp2,rs,rt); break; case LOADBU_STUB: emit_ldrb_dualindexed(temp2,rs,rt); break; @@ -1438,10 +1462,10 @@ static void do_readstub(int n) int cc=get_reg(i_regmap,CCREG); if(cc<0) emit_loadreg(CCREG,2); - emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2); - emit_call(handler); + emit_addimm(cc<0?2:cc,(int)stubs[n].d,2); + emit_far_call(handler); // (no cycle reload after read) - if(itype[i]==C1LS||itype[i]==C2LS||(rt>=0&&rt1[i]!=0)) { + if(dops[i].itype==C1LS||dops[i].itype==C2LS||(rt>=0&&dops[i].rt1!=0)) { loadstore_extend(type,0,rt); } if(restore_jump) @@ -1450,27 +1474,25 @@ static void do_readstub(int n) emit_jmp(stubs[n].retaddr); } -static void inline_readstub(enum stub_type type, int i, u_int addr, signed char regmap[], int target, int adj, u_int reglist) +static void inline_readstub(enum stub_type type, int i, u_int addr, + const signed char regmap[], int target, int adj, u_int reglist) { int rs=get_reg(regmap,target); int rt=get_reg(regmap,target); - if(rs<0) rs=get_reg(regmap,-1); + if(rs<0) rs=get_reg_temp(regmap); assert(rs>=0); u_int is_dynamic=0; uintptr_t host_addr = 0; void *handler; int cc=get_reg(regmap,CCREG); - //if(pcsx_direct_read(type,addr,CLOCK_ADJUST(adj+1),cc,target?rs:-1,rt)) + //if(pcsx_direct_read(type,addr,adj,cc,target?rs:-1,rt)) // return; handler = get_direct_memhandler(mem_rtab, addr, type, &host_addr); if (handler == NULL) { - if(rt<0||rt1[i]==0) + if(rt<0||dops[i].rt1==0) return; - if (addr != host_addr) { - if (host_addr >= 0x100000000ull) - abort(); // ROREG not implemented - emit_movimm_from(addr, rs, host_addr, rs); - } + if (addr != host_addr) + emit_movimm_from64(addr, rs, host_addr, rs); switch(type) { case LOADB_STUB: emit_movsbl_indexed(0,rs,rt); break; case LOADBU_STUB: emit_movzbl_indexed(0,rs,rt); break; @@ -1481,8 +1503,8 @@ static void inline_readstub(enum stub_type type, int i, u_int addr, signed char } return; } - is_dynamic=pcsxmem_is_handler_dynamic(addr); - if(is_dynamic) { + is_dynamic = pcsxmem_is_handler_dynamic(addr); + if (is_dynamic) { if(type==LOADB_STUB||type==LOADBU_STUB) handler=jump_handler_read8; if(type==LOADH_STUB||type==LOADHU_STUB) @@ -1492,7 +1514,7 @@ static void inline_readstub(enum stub_type type, int i, u_int addr, signed char } // call a memhandler - if(rt>=0&&rt1[i]!=0) + if(rt>=0&&dops[i].rt1!=0) reglist&=~(1<>12] << 1; - emit_adrp((void *)l1, 1); - emit_addimm64(1, l1 & 0xfff, 1); + intptr_t offset = (l1 & ~0xfffl) - ((intptr_t)out & ~0xfffl); + if (-4294967296l <= offset && offset < 4294967296l) { + emit_adrp((void *)l1, 1); + emit_addimm64(1, l1 & 0xfff, 1); + } + else + emit_movimm64(l1, 1); } else - emit_call(do_memhandler_pre); + emit_far_call(do_memhandler_pre); - emit_call(handler); + emit_far_call(handler); // (no cycle reload after read) - if(rt>=0&&rt1[i]!=0) + if(rt>=0&&dops[i].rt1!=0) loadstore_extend(type, 0, rt); restore_regs(reglist); } @@ -1529,10 +1556,10 @@ static void do_writestub(int n) u_int reglist=stubs[n].e; signed char *i_regmap=i_regs->regmap; int rt,r; - if(itype[i]==C1LS||itype[i]==C2LS) { + if(dops[i].itype==C1LS||dops[i].itype==C2LS) { rt=get_reg(i_regmap,r=FTEMP); }else{ - rt=get_reg(i_regmap,r=rs2[i]); + rt=get_reg(i_regmap,r=dops[i].rs2); } assert(rs>=0); assert(rt>=0); @@ -1578,7 +1605,6 @@ static void do_writestub(int n) emit_jmp(stubs[n].retaddr); // return address (invcode check) set_jump_target(handler_jump, out); - // TODO FIXME: regalloc should prefer callee-saved regs if(!regs_saved) save_regs(reglist); void *handler=NULL; @@ -1597,10 +1623,10 @@ static void do_writestub(int n) int cc=get_reg(i_regmap,CCREG); if(cc<0) emit_loadreg(CCREG,2); - emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2); + emit_addimm(cc<0?2:cc,(int)stubs[n].d,2); // returns new cycle_count - emit_call(handler); - emit_addimm(0,-CLOCK_ADJUST((int)stubs[n].d+1),cc<0?2:cc); + emit_far_call(handler); + emit_addimm(0,-(int)stubs[n].d,cc<0?2:cc); if(cc<0) emit_storereg(CCREG,2); if(restore_jump) @@ -1609,20 +1635,18 @@ static void do_writestub(int n) emit_jmp(stubs[n].retaddr); } -static void inline_writestub(enum stub_type type, int i, u_int addr, signed char regmap[], int target, int adj, u_int reglist) +static void inline_writestub(enum stub_type type, int i, u_int addr, + const signed char regmap[], int target, int adj, u_int reglist) { - int rs = get_reg(regmap,-1); + int rs = get_reg_temp(regmap); int rt = get_reg(regmap,target); assert(rs >= 0); assert(rt >= 0); uintptr_t host_addr = 0; void *handler = get_direct_memhandler(mem_wtab, addr, type, &host_addr); if (handler == NULL) { - if (addr != host_addr) { - if (host_addr >= 0x100000000ull) - abort(); // ROREG not implemented - emit_movimm_from(addr, rs, host_addr, rs); - } + if (addr != host_addr) + emit_movimm_from64(addr, rs, host_addr, rs); switch (type) { case STOREB_STUB: emit_writebyte_indexed(rt, 0, rs); break; case STOREH_STUB: emit_writehword_indexed(rt, 0, rs); break; @@ -1640,151 +1664,26 @@ static void inline_writestub(enum stub_type type, int i, u_int addr, signed char cc = cc_use = get_reg(regmap, CCREG); if (cc < 0) emit_loadreg(CCREG, (cc_use = 2)); - emit_addimm(cc_use, CLOCK_ADJUST(adj+1), 2); + emit_addimm(cc_use, adj, 2); - emit_call(do_memhandler_pre); - emit_call(handler); - emit_call(do_memhandler_post); - emit_addimm(0, -CLOCK_ADJUST(adj+1), cc_use); + emit_far_call(do_memhandler_pre); + emit_far_call(handler); + emit_far_call(do_memhandler_post); + emit_addimm(0, -adj, cc_use); if (cc < 0) emit_storereg(CCREG, cc_use); restore_regs(reglist); } -static int verify_code_arm64(const void *source, const void *copy, u_int size) -{ - int ret = memcmp(source, copy, size); - //printf("%s %p,%#x = %d\n", __func__, source, size, ret); - return ret; -} - -// this output is parsed by verify_dirty, get_bounds, isclean, get_clean_addr -static void do_dirty_stub_base(u_int vaddr) -{ - assert(slen <= MAXBLOCK); - emit_loadlp_ofs(0, 0); // ldr x1, source - emit_loadlp_ofs(0, 1); // ldr x2, copy - emit_movz(slen*4, 2); - emit_call(verify_code_arm64); - void *jmp = out; - emit_cbz(0, 0); - emit_movz(vaddr & 0xffff, 0); - emit_movk_lsl16(vaddr >> 16, 0); - emit_call(get_addr); - emit_jmpreg(0); - set_jump_target(jmp, out); -} - -static void assert_dirty_stub(const u_int *ptr) -{ - assert((ptr[0] & 0xff00001f) == 0x58000000); // ldr x0, source - assert((ptr[1] & 0xff00001f) == 0x58000001); // ldr x1, copy - assert((ptr[2] & 0xffe0001f) == 0x52800002); // movz w2, #slen*4 - assert( ptr[8] == 0xd61f0000); // br x0 -} - -static void set_loadlp(u_int *loadl, void *lit) -{ - uintptr_t ofs = (u_char *)lit - (u_char *)loadl; - assert((*loadl & ~0x1f) == 0x58000000); - assert((ofs & 3) == 0); - assert(ofs < 0x100000); - *loadl |= (ofs >> 2) << 5; -} - -static void do_dirty_stub_emit_literals(u_int *loadlps) -{ - set_loadlp(&loadlps[0], out); - output_w64((uintptr_t)source); - set_loadlp(&loadlps[1], out); - output_w64((uintptr_t)copy); -} - -static void *do_dirty_stub(int i) -{ - assem_debug("do_dirty_stub %x\n",start+i*4); - u_int *loadlps = (void *)out; - do_dirty_stub_base(start + i*4); - void *entry = out; - load_regs_entry(i); - if (entry == out) - entry = instr_addr[i]; - emit_jmp(instr_addr[i]); - do_dirty_stub_emit_literals(loadlps); - return entry; -} - -static void do_dirty_stub_ds(void) -{ - u_int *loadlps = (void *)out; - do_dirty_stub_base(start + 1); - void *lit_jumpover = out; - emit_jmp(out + 8*2); - do_dirty_stub_emit_literals(loadlps); - set_jump_target(lit_jumpover, out); -} - -static uint64_t get_from_ldr_literal(const u_int *i) -{ - signed int ofs; - assert((i[0] & 0xff000000) == 0x58000000); - ofs = i[0] << 8; - ofs >>= 5+8; - return *(uint64_t *)(i + ofs); -} - -static uint64_t get_from_movz(const u_int *i) -{ - assert((i[0] & 0x7fe00000) == 0x52800000); - return (i[0] >> 5) & 0xffff; -} - -// Find the "clean" entry point from a "dirty" entry point -// by skipping past the call to verify_code -static void *get_clean_addr(u_int *addr) -{ - assert_dirty_stub(addr); - return addr + 9; -} - -static int verify_dirty(const u_int *ptr) -{ - const void *source, *copy; - u_int len; - assert_dirty_stub(ptr); - source = (void *)get_from_ldr_literal(&ptr[0]); // ldr x1, source - copy = (void *)get_from_ldr_literal(&ptr[1]); // ldr x1, copy - len = get_from_movz(&ptr[2]); // movz w3, #slen*4 - return !memcmp(source, copy, len); -} - -static int isclean(void *addr) -{ - const u_int *ptr = addr; - if ((*ptr >> 24) == 0x58) { // the only place ldr (literal) is used - assert_dirty_stub(ptr); - return 0; - } - return 1; -} - -// get source that block at addr was compiled from (host pointers) -static void get_bounds(void *addr, u_char **start, u_char **end) -{ - const u_int *ptr = addr; - assert_dirty_stub(ptr); - *start = (u_char *)get_from_ldr_literal(&ptr[0]); // ldr x1, source - *end = *start + get_from_movz(&ptr[2]); // movz w3, #slen*4 -} - /* Special assem */ -static void c2op_prologue(u_int op,u_int reglist) +static void c2op_prologue(u_int op, int i, const struct regstat *i_regs, u_int reglist) { save_load_regs_all(1, reglist); + cop2_do_stall_check(op, i, i_regs, 0); #ifdef PCNT emit_movimm(op, 0); - emit_call(pcnt_gte_start); + emit_far_call(pcnt_gte_start); #endif // pointer to cop2 regs emit_addimm64(FP, (u_char *)&psxRegs.CP2D.r[0] - (u_char *)&dynarec_local, 0); @@ -1794,12 +1693,12 @@ static void c2op_epilogue(u_int op,u_int reglist) { #ifdef PCNT emit_movimm(op, 0); - emit_call(pcnt_gte_end); + emit_far_call(pcnt_gte_end); #endif save_load_regs_all(0, reglist); } -static void c2op_assemble(int i,struct regstat *i_regs) +static void c2op_assemble(int i, const struct regstat *i_regs) { u_int c2op=source[i]&0x3f; u_int hr,reglist_full=0,reglist; @@ -1814,17 +1713,17 @@ static void c2op_assemble(int i,struct regstat *i_regs) need_ir=(gte_unneeded[i+1]&0xe00)!=0xe00; assem_debug("gte op %08x, unneeded %016lx, need_flags %d, need_ir %d\n", source[i],gte_unneeded[i+1],need_flags,need_ir); - if(new_dynarec_hacks&NDHACK_GTE_NO_FLAGS) + if(HACK_ENABLED(NDHACK_GTE_NO_FLAGS)) need_flags=0; //int shift = (source[i] >> 19) & 1; //int lm = (source[i] >> 10) & 1; switch(c2op) { default: (void)need_ir; - c2op_prologue(c2op,reglist); + c2op_prologue(c2op, i, i_regs, reglist); emit_movimm(source[i],1); // opcode emit_writeword(1,&psxRegs.code); - emit_call(need_flags?gte_handlers[c2op]:gte_handlers_nf[c2op]); + emit_far_call(need_flags?gte_handlers[c2op]:gte_handlers_nf[c2op]); break; } c2op_epilogue(c2op,reglist); @@ -1872,21 +1771,21 @@ static void c2op_mfc2_29_assemble(signed char tl, signed char temp) host_tempreg_release(); } -static void multdiv_assemble_arm64(int i,struct regstat *i_regs) +static void multdiv_assemble_arm64(int i, const struct regstat *i_regs) { // case 0x18: MULT // case 0x19: MULTU // case 0x1A: DIV // case 0x1B: DIVU - if(rs1[i]&&rs2[i]) + if(dops[i].rs1&&dops[i].rs2) { - switch(opcode2[i]) + switch(dops[i].opcode2) { case 0x18: // MULT case 0x19: // MULTU { - signed char m1=get_reg(i_regs->regmap,rs1[i]); - signed char m2=get_reg(i_regs->regmap,rs2[i]); + signed char m1=get_reg(i_regs->regmap,dops[i].rs1); + signed char m2=get_reg(i_regs->regmap,dops[i].rs2); signed char hi=get_reg(i_regs->regmap,HIREG); signed char lo=get_reg(i_regs->regmap,LOREG); assert(m1>=0); @@ -1894,7 +1793,7 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) assert(hi>=0); assert(lo>=0); - if(opcode2[i]==0x18) // MULT + if(dops[i].opcode2==0x18) // MULT emit_smull(m1,m2,hi); else // MULTU emit_umull(m1,m2,hi); @@ -1906,8 +1805,8 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) case 0x1A: // DIV case 0x1B: // DIVU { - signed char numerator=get_reg(i_regs->regmap,rs1[i]); - signed char denominator=get_reg(i_regs->regmap,rs2[i]); + signed char numerator=get_reg(i_regs->regmap,dops[i].rs1); + signed char denominator=get_reg(i_regs->regmap,dops[i].rs2); signed char quotient=get_reg(i_regs->regmap,LOREG); signed char remainder=get_reg(i_regs->regmap,HIREG); assert(numerator>=0); @@ -1915,7 +1814,7 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) assert(quotient>=0); assert(remainder>=0); - if (opcode2[i] == 0x1A) // DIV + if (dops[i].opcode2 == 0x1A) // DIV emit_sdiv(numerator,denominator,quotient); else // DIVU emit_udiv(numerator,denominator,quotient); @@ -1923,7 +1822,7 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) // div 0 quotient (remainder is already correct) host_tempreg_acquire(); - if (opcode2[i] == 0x1A) // DIV + if (dops[i].opcode2 == 0x1A) // DIV emit_sub_asrimm(0,numerator,31,HOST_TEMPREG); else emit_movimm(~0,HOST_TEMPREG); @@ -1940,15 +1839,15 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) { signed char hr=get_reg(i_regs->regmap,HIREG); signed char lr=get_reg(i_regs->regmap,LOREG); - if ((opcode2[i]==0x1A || opcode2[i]==0x1B) && rs2[i]==0) // div 0 + if ((dops[i].opcode2==0x1A || dops[i].opcode2==0x1B) && dops[i].rs2==0) // div 0 { - if (rs1[i]) { - signed char numerator = get_reg(i_regs->regmap, rs1[i]); + if (dops[i].rs1) { + signed char numerator = get_reg(i_regs->regmap, dops[i].rs1); assert(numerator >= 0); if (hr >= 0) emit_mov(numerator,hr); if (lr >= 0) { - if (opcode2[i] == 0x1A) // DIV + if (dops[i].opcode2 == 0x1A) // DIV emit_sub_asrimm(0,numerator,31,lr); else emit_movimm(~0,lr); @@ -1973,7 +1872,7 @@ static void do_jump_vaddr(u_int rs) { if (rs != 0) emit_mov(rs, 0); - emit_call(get_addr_ht); + emit_far_call(ndrc_get_addr_ht); emit_jmpreg(0); } @@ -2017,51 +1916,60 @@ static void do_miniht_insert(u_int return_address,u_int rt,int temp) { emit_writeword(rt,&mini_ht[(return_address&0xFF)>>3][0]); } -static void mark_clear_cache(void *target) +static unused void clear_cache_arm64(char *start, char *end) { - u_long offset = (u_char *)target - translation_cache; - u_int mask = 1u << ((offset >> 12) & 31); - if (!(needs_clear_cache[offset >> 17] & mask)) { - char *start = (char *)((u_long)target & ~4095ul); - start_tcache_write(start, start + 4096); - needs_clear_cache[offset >> 17] |= mask; + // Don't rely on GCC's __clear_cache implementation, as it caches + // icache/dcache cache line sizes, that can vary between cores on + // big.LITTLE architectures. + uint64_t addr, ctr_el0; + static size_t icache_line_size = 0xffff, dcache_line_size = 0xffff; + size_t isize, dsize; + + __asm__ volatile("mrs %0, ctr_el0" : "=r"(ctr_el0)); + isize = 4 << ((ctr_el0 >> 0) & 0xf); + dsize = 4 << ((ctr_el0 >> 16) & 0xf); + + // use the global minimum cache line size + icache_line_size = isize = icache_line_size < isize ? icache_line_size : isize; + dcache_line_size = dsize = dcache_line_size < dsize ? dcache_line_size : dsize; + + /* If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification is + not required for instruction to data coherence. */ + if ((ctr_el0 & (1 << 28)) == 0x0) { + addr = (uint64_t)start & ~(uint64_t)(dsize - 1); + for (; addr < (uint64_t)end; addr += dsize) + // use "civac" instead of "cvau", as this is the suggested workaround for + // Cortex-A53 errata 819472, 826319, 827319 and 824069. + __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory"); } -} + __asm__ volatile("dsb ish" : : : "memory"); -// Clearing the cache is rather slow on ARM Linux, so mark the areas -// that need to be cleared, and then only clear these areas once. -static void do_clear_cache() -{ - int i,j; - for (i=0;i<(1<<(TARGET_SIZE_2-17));i++) - { - u_int bitmap=needs_clear_cache[i]; - if(bitmap) { - u_char *start, *end; - for(j=0;j<32;j++) - { - if(bitmap&(1<tramp.f - (u_char *)&ndrc->tramp.ops; + struct tramp_insns *ops = NDRC_WRITE_OFFSET(ndrc->tramp.ops); + size_t i; + assert(!(diff & 3)); + start_tcache_write(ops, (u_char *)ops + sizeof(ndrc->tramp.ops)); + for (i = 0; i < ARRAY_SIZE(ndrc->tramp.ops); i++) { + ops[i].ldr = 0x58000000 | imm19_rt(diff >> 2, 17); // ldr x17, [=val] + ops[i].br = 0xd61f0000 | rm_rn_rd(0, 17, 0); // br x17 + } + end_tcache_write(ops, (u_char *)ops + sizeof(ndrc->tramp.ops)); } // vim:shiftwidth=2:expandtab