From d1e4ebd9988a9a5d9fb38b89f19e24b9ab6029d7 Mon Sep 17 00:00:00 2001 From: notaz Date: Mon, 8 Nov 2021 22:26:05 +0000 Subject: [PATCH] drc: arm64 wip --- libpcsxcore/new_dynarec/assem_arm.c | 70 +- libpcsxcore/new_dynarec/assem_arm64.c | 1277 ++++++++++++----- libpcsxcore/new_dynarec/assem_arm64.h | 12 +- libpcsxcore/new_dynarec/linkage_arm.S | 8 - libpcsxcore/new_dynarec/linkage_arm64.S | 139 +- libpcsxcore/new_dynarec/linkage_offsets.h | 2 +- libpcsxcore/new_dynarec/new_dynarec.c | 140 +- libpcsxcore/new_dynarec/patches/trace_drc_chk | 183 +-- libpcsxcore/new_dynarec/pcsxmem_inline.c | 4 + 9 files changed, 1301 insertions(+), 534 deletions(-) diff --git a/libpcsxcore/new_dynarec/assem_arm.c b/libpcsxcore/new_dynarec/assem_arm.c index 45a2f086..ed00103f 100644 --- a/libpcsxcore/new_dynarec/assem_arm.c +++ b/libpcsxcore/new_dynarec/assem_arm.c @@ -424,6 +424,13 @@ static u_int genjmp(u_int addr) return ((u_int)offset>>2)&0xffffff; } +static unused void emit_breakpoint(void) +{ + assem_debug("bkpt #0\n"); + //output_w32(0xe1200070); + output_w32(0xe7f001f0); +} + static void emit_mov(int rs,int rt) { assem_debug("mov %s,%s\n",regname[rt],regname[rs]); @@ -1022,7 +1029,7 @@ static void emit_set_if_carry32(int rs1, int rs2, int rt) static void emit_call(const void *a_) { int a = (int)a_; - assem_debug("bl %x (%x+%x)%s\n",a,(int)out,a-(int)out-8,func_name(a)); + assem_debug("bl %x (%x+%x)%s\n",a,(int)out,a-(int)out-8,func_name(a_)); u_int offset=genjmp(a); output_w32(0xeb000000|offset); } @@ -1030,7 +1037,7 @@ static void emit_call(const void *a_) static void emit_jmp(const void *a_) { int a = (int)a_; - assem_debug("b %x (%x+%x)%s\n",a,(int)out,a-(int)out-8,func_name(a)); + assem_debug("b %x (%x+%x)%s\n",a,(int)out,a-(int)out-8,func_name(a_)); u_int offset=genjmp(a); output_w32(0xea000000|offset); } @@ -1380,21 +1387,6 @@ static void emit_rsbimm(int rs, int imm, int rt) output_w32(0xe2600000|rd_rn_rm(rt,rs,0)|armval); } -// Load 2 immediates optimizing for small code size -static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2) -{ - emit_movimm(imm1,rt1); - u_int armval; - if(genimm(imm2-imm1,&armval)) { - assem_debug("add %s,%s,#%d\n",regname[rt2],regname[rt1],imm2-imm1); - output_w32(0xe2800000|rd_rn_rm(rt2,rt1,0)|armval); - }else if(genimm(imm1-imm2,&armval)) { - assem_debug("sub %s,%s,#%d\n",regname[rt2],regname[rt1],imm1-imm2); - output_w32(0xe2400000|rd_rn_rm(rt2,rt1,0)|armval); - } - else emit_movimm(imm2,rt2); -} - // Conditionally select one of two immediates, optimizing for small code size // This will only be called if HAVE_CMOV_IMM is defined static void emit_cmov2imm_e_ne_compact(int imm1,int imm2,u_int rt) @@ -1601,6 +1593,13 @@ static void emit_extjump2(u_char *addr, u_int target, void *linker) emit_jmp(linker); } +static void check_extjump2(void *src) +{ + u_int *ptr = src; + assert((ptr[1] & 0x0fff0000) == 0x059f0000); // ldr rx, [pc, #ofs] + (void)ptr; +} + // put rt_val into rt, potentially making use of rs with value rs_val static void emit_movimm_from(u_int rs_val,int rs,u_int rt_val,int rt) { @@ -2029,19 +2028,7 @@ static void do_unalignedwritestub(int n) #endif } -static void do_invstub(int n) -{ - literal_pool(20); - u_int reglist=stubs[n].a; - set_jump_target(stubs[n].addr, out); - save_regs(reglist); - if(stubs[n].b!=0) emit_mov(stubs[n].b,0); - emit_call(&invalidate_addr); - restore_regs(reglist); - emit_jmp(stubs[n].retaddr); // return address -} - -// this output is parsed by verify_dirty, get_bounds +// this output is parsed by verify_dirty, get_bounds, isclean, get_clean_addr static void do_dirty_stub_emit_args(u_int arg0) { #ifndef HAVE_ARMV7 @@ -2196,7 +2183,7 @@ static void loadlr_assemble_arm(int i,struct regstat *i_regs) int offset; void *jaddr=0; int memtarget=0,c=0; - int fastload_reg_override=0; + int fastio_reg_override=-1; u_int hr,reglist=0; tl=get_reg(i_regs->regmap,rt1[i]); s=get_reg(i_regs->regmap,rs1[i]); @@ -2224,12 +2211,13 @@ static void loadlr_assemble_arm(int i,struct regstat *i_regs) }else{ emit_andimm(addr,0xFFFFFFF8,temp2); // LDL/LDR } - jaddr=emit_fastpath_cmp_jump(i,temp2,&fastload_reg_override); + jaddr=emit_fastpath_cmp_jump(i,temp2,&fastio_reg_override); } else { if(ram_offset&&memtarget) { + host_tempreg_acquire(); emit_addimm(temp2,ram_offset,HOST_TEMPREG); - fastload_reg_override=HOST_TEMPREG; + fastio_reg_override=HOST_TEMPREG; } if (opcode[i]==0x22||opcode[i]==0x26) { emit_movimm(((constmap[i][s]+offset)<<3)&24,temp); // LWL/LWR @@ -2240,8 +2228,9 @@ static void loadlr_assemble_arm(int i,struct regstat *i_regs) if (opcode[i]==0x22||opcode[i]==0x26) { // LWL/LWR if(!c||memtarget) { int a=temp2; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_readword_indexed(0,a,temp2); + if(fastio_reg_override==HOST_TEMPREG) host_tempreg_release(); if(jaddr) add_stub_r(LOADW_STUB,jaddr,out,i,temp2,i_regs,ccadj[i],reglist); } else @@ -2555,6 +2544,11 @@ static void multdiv_assemble_arm(int i,struct regstat *i_regs) } #define multdiv_assemble multdiv_assemble_arm +static void do_jump_vaddr(int rs) +{ + emit_jmp(jump_vaddr_reg[rs]); +} + static void do_preload_rhash(int r) { // Don't need this for ARM. On x86, this puts the value 0xf8 into the // register. On ARM the hash can be done with a single instruction (below) @@ -2577,11 +2571,11 @@ static void do_miniht_jump(int rs,int rh,int ht) { emit_cmp(rh,rs); emit_ldreq_indexed(ht,4,15); #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK - emit_mov(rs,7); - emit_jmp(jump_vaddr_reg[7]); - #else - emit_jmp(jump_vaddr_reg[rs]); + if(rs!=7) + emit_mov(rs,7); + rs=7; #endif + do_jump_vaddr(rs); } static void do_miniht_insert(u_int return_address,int rt,int temp) { diff --git a/libpcsxcore/new_dynarec/assem_arm64.c b/libpcsxcore/new_dynarec/assem_arm64.c index fabd7dba..a0c628b5 100644 --- a/libpcsxcore/new_dynarec/assem_arm64.c +++ b/libpcsxcore/new_dynarec/assem_arm64.c @@ -1,7 +1,8 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Mupen64plus/PCSX - assem_arm64.c * * Copyright (C) 2009-2011 Ari64 * - * Copyright (C) 2010-2021 notaz * + * Copyright (C) 2009-2018 Gillou68310 * + * Copyright (C) 2021 notaz * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * @@ -27,33 +28,50 @@ u_char *translation_cache; #else u_char translation_cache[1 << TARGET_SIZE_2] __attribute__((aligned(4096))); #endif +static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)]; #define CALLER_SAVE_REGS 0x0007ffff #define unused __attribute__((unused)) -static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)]; - -//void indirect_jump_indexed(); -//void indirect_jump(); -void do_interrupt(); -//void jump_vaddr_r0(); - -void * const jump_vaddr_reg[32]; +void do_memhandler_pre(); +void do_memhandler_post(); /* Linker */ - -static void set_jump_target(void *addr, void *target_) +static void set_jump_target(void *addr, void *target) { - assert(0); + u_int *ptr = addr; + intptr_t offset = (u_char *)target - (u_char *)addr; + + if((*ptr&0xFC000000)==0x14000000) { + assert(offset>=-134217728LL&&offset<134217728LL); + *ptr=(*ptr&0xFC000000)|((offset>>2)&0x3ffffff); + } + else if((*ptr&0xff000000)==0x54000000) { + // Conditional branch are limited to +/- 1MB + // block max size is 256k so branching beyond the +/- 1MB limit + // should only happen when jumping to an already compiled block (see add_link) + // a workaround would be to do a trampoline jump via a stub at the end of the block + assert(offset>=-1048576LL&&offset<1048576LL); + *ptr=(*ptr&0xFF00000F)|(((offset>>2)&0x7ffff)<<5); + } + else if((*ptr&0x9f000000)==0x10000000) { //adr + // generated by do_miniht_insert + assert(offset>=-1048576LL&&offset<1048576LL); + *ptr=(*ptr&0x9F00001F)|(offset&0x3)<<29|((offset>>2)&0x7ffff)<<5; + } + else + assert(0); // should not happen } // from a pointer to external jump stub (which was produced by emit_extjump2) // find where the jumping insn is static void *find_extjump_insn(void *stub) { - assert(0); - return NULL; + int *ptr = (int *)stub + 2; + assert((*ptr&0x9f000000) == 0x10000000); // adr + int offset = (((signed int)(*ptr<<8)>>13)<<2)|((*ptr>>29)&0x3); + return ptr + offset / 4; } // find where external branch is liked to using addr of it's stub: @@ -62,9 +80,9 @@ static void *find_extjump_insn(void *stub) // return addr where that branch jumps to static void *get_pointer(void *stub) { - //printf("get_pointer(%x)\n",(int)stub); - assert(0); - return NULL; + int *i_ptr = find_extjump_insn(stub); + assert((*i_ptr&0xfc000000) == 0x14000000); // b + return (u_char *)i_ptr+(((signed int)(*i_ptr<<6)>>6)<<2); } // Find the "clean" entry point from a "dirty" entry point @@ -81,18 +99,36 @@ static int verify_dirty(u_int *ptr) return 0; } -// This doesn't necessarily find all clean entry points, just -// guarantees that it's not dirty static int isclean(void *addr) { - assert(0); - return 0; + u_int *ptr = addr; + return (*ptr >> 24) != 0x58; // the only place ldr (literal) is used +} + +static uint64_t get_from_ldr_literal(const u_int *i) +{ + signed int ofs; + assert((i[0] & 0xff000000) == 0x58000000); + ofs = i[0] << 8; + ofs >>= 5+8; + return *(uint64_t *)(i + ofs); +} + +static uint64_t get_from_movz(const u_int *i) +{ + assert((i[0] & 0x7fe00000) == 0x52800000); + return (i[0] >> 5) & 0xffff; } // get source that block at addr was compiled from (host pointers) static void get_bounds(void *addr, u_char **start, u_char **end) { - assert(0); + const u_int *ptr = addr; + assert((ptr[0] & 0xff00001f) == 0x58000001); // ldr x1, source + assert((ptr[1] & 0xff00001f) == 0x58000002); // ldr x2, copy + assert((ptr[2] & 0xffe0001f) == 0x52800003); // movz w3, #slen*4 + *start = (u_char *)get_from_ldr_literal(&ptr[0]); + *end = *start + get_from_movz(&ptr[2]); } // Allocate a specific ARM register. @@ -128,10 +164,27 @@ static void alloc_cc(struct regstat *cur,int i) /* Assembler */ static unused const char *regname[32] = { - "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", - "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" - "ip0", "ip1", "r18", "r19", "r20", "r21", "r22", "r23" - "r24", "r25", "r26", "r27", "r28", "fp", "lr", "sp" + "w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7", + "w8", "w9", "w10", "w11", "w12", "w13", "w14", "w15", + "ip0", "ip1", "w18", "w19", "w20", "w21", "w22", "w23", + "w24", "w25", "w26", "w27", "w28", "wfp", "wlr", "wsp" +}; + +static unused const char *regname64[32] = { + "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + "ip0", "ip1", "x18", "x19", "x20", "x21", "x22", "x23", + "x24", "x25", "x26", "x27", "x28", "fp", "lr", "sp" +}; + +enum { + COND_EQ, COND_NE, COND_CS, COND_CC, COND_MI, COND_PL, COND_VS, COND_VC, + COND_HI, COND_LS, COND_GE, COND_LT, COND_GT, COND_LE, COND_AW, COND_NV +}; + +static unused const char *condname[16] = { + "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", + "hi", "ls", "ge", "lt", "gt", "le", "aw", "nv" }; static void output_w32(u_int word) @@ -140,21 +193,38 @@ static void output_w32(u_int word) out += 4; } +static void output_w64(uint64_t dword) +{ + *((uint64_t *)out) = dword; + out+=8; +} + +/* static u_int rm_rd(u_int rm, u_int rd) { assert(rm < 31); assert(rd < 31); return (rm << 16) | rd; } +*/ static u_int rm_rn_rd(u_int rm, u_int rn, u_int rd) { - assert(rm < 31); - assert(rn < 31); - assert(rd < 31); + assert(rm < 32); + assert(rn < 32); + assert(rd < 32); return (rm << 16) | (rn << 5) | rd; } +static u_int imm7_rt2_rn_rt(u_int imm7, u_int rt2, u_int rn, u_int rt) +{ + assert(imm7 < 0x80); + assert(rt2 < 31); + assert(rn < 32); + assert(rt < 31); + return (imm7 << 15) | (rt2 << 10) | (rn << 5) | rt; +} + static u_int rm_imm6_rn_rd(u_int rm, u_int imm6, u_int rn, u_int rd) { assert(imm6 <= 63); @@ -171,102 +241,217 @@ static u_int imm16_rd(u_int imm16, u_int rd) static u_int imm12_rn_rd(u_int imm12, u_int rn, u_int rd) { assert(imm12 < 0x1000); + assert(rn < 32); + assert(rd < 32); + return (imm12 << 10) | (rn << 5) | rd; +} + +static u_int imm9_rn_rt(u_int imm9, u_int rn, u_int rd) +{ + assert(imm9 < 0x200); assert(rn < 31); assert(rd < 31); - return (imm12 << 10) | (rn << 5) | rd; + return (imm9 << 12) | (rn << 5) | rd; } -#pragma GCC diagnostic ignored "-Wunused-function" -static u_int genjmp(u_char *addr) +static u_int imm19_rt(u_int imm19, u_int rt) +{ + assert(imm19 < 0x80000); + assert(rt < 31); + return (imm19 << 5) | rt; +} + +static u_int n_immr_imms_rn_rd(u_int n, u_int immr, u_int imms, u_int rn, u_int rd) +{ + assert(n < 2); + assert(immr < 0x40); + assert(imms < 0x40); + assert(rn < 32); + assert(rd < 32); + return (n << 22) | (immr << 16) | (imms << 10) | (rn << 5) | rd; +} + +static u_int genjmp(const u_char *addr) { intptr_t offset = addr - out; + if ((uintptr_t)addr < 3) return 0; // a branch that will be patched later if (offset < -134217728 || offset > 134217727) { - if ((uintptr_t)addr > 2) { - SysPrintf("%s: out of range: %08x\n", __func__, offset); - exit(1); - } + SysPrintf("%s: out of range: %p %lx\n", __func__, addr, offset); + abort(); return 0; } - return ((u_int)offset >> 2) & 0x01ffffff; + return ((u_int)offset >> 2) & 0x03ffffff; } -static u_int genjmpcc(u_char *addr) +static u_int genjmpcc(const u_char *addr) { intptr_t offset = addr - out; + if ((uintptr_t)addr < 3) return 0; if (offset < -1048576 || offset > 1048572) { - if ((uintptr_t)addr > 2) { - SysPrintf("%s: out of range: %08x\n", __func__, offset); - exit(1); - } + SysPrintf("%s: out of range: %p %lx\n", __func__, addr, offset); + abort(); + return 0; + } + return ((u_int)offset >> 2) & 0x7ffff; +} + +static uint32_t is_mask(u_int value) +{ + return value && ((value + 1) & value) == 0; +} + +// This function returns true if the argument contains a +// non-empty sequence of ones (possibly rotated) with the remainder zero. +static uint32_t is_rotated_mask(u_int value) +{ + if (value == 0) return 0; + if (is_mask((value - 1) | value)) + return 1; + return is_mask((~value - 1) | ~value); +} + +static void gen_logical_imm(u_int value, u_int *immr, u_int *imms) +{ + int lzeros, tzeros, ones; + assert(value != 0); + if (is_mask((value - 1) | value)) { + lzeros = __builtin_clz(value); + tzeros = __builtin_ctz(value); + ones = 32 - lzeros - tzeros; + *immr = (32 - tzeros) & 31; + *imms = ones - 1; + return; } - return ((u_int)offset >> 2) & 0xfffff; + value = ~value; + if (is_mask((value - 1) | value)) { + lzeros = __builtin_clz(value); + tzeros = __builtin_ctz(value); + ones = 32 - lzeros - tzeros; + *immr = 31 - tzeros; + *imms = 31 - ones; + return; + } + assert(0); } static void emit_mov(u_int rs, u_int rt) { assem_debug("mov %s,%s\n", regname[rt], regname[rs]); - output_w32(0x2a0003e0 | rm_rd(rs, rt)); + output_w32(0x2a000000 | rm_rn_rd(rs, WZR, rt)); +} + +static void emit_mov64(u_int rs, u_int rt) +{ + assem_debug("mov %s,%s\n", regname64[rt], regname64[rs]); + output_w32(0xaa000000 | rm_rn_rd(rs, WZR, rt)); } static void emit_movs(u_int rs, u_int rt) { + assert(0); // misleading assem_debug("movs %s,%s\n", regname[rt], regname[rs]); output_w32(0x31000000 | imm12_rn_rd(0, rs, rt)); } static void emit_add(u_int rs1, u_int rs2, u_int rt) { - assem_debug("add %s, %s, %s\n", regname[rt], regname[rs1], regname[rs2]); - output_w32(0x0b000000 | rm_imm6_rn_rd(rs2, 0, rs1, rt)); + assem_debug("add %s,%s,%s\n", regname[rt], regname[rs1], regname[rs2]); + output_w32(0x0b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_sbc(u_int rs1,u_int rs2,u_int rt) +static void emit_add64(u_int rs1, u_int rs2, u_int rt) { - assem_debug("sbc %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + assem_debug("add %s,%s,%s\n", regname64[rt], regname64[rs1], regname64[rs2]); + output_w32(0x8b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_neg(u_int rs, u_int rt) +#pragma GCC diagnostic ignored "-Wunused-function" +static void emit_adds(u_int rs1, u_int rs2, u_int rt) { - assem_debug("rsb %s,%s,#0\n",regname[rt],regname[rs]); - assert(0); + assem_debug("adds %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); + output_w32(0x2b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_negs(u_int rs, u_int rt) +static void emit_adds64(u_int rs1, u_int rs2, u_int rt) { - assem_debug("rsbs %s,%s,#0\n",regname[rt],regname[rs]); - assert(0); + assem_debug("adds %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); + output_w32(0xab000000 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_neg(u_int rs, u_int rt) +{ + assem_debug("neg %s,%s\n",regname[rt],regname[rs]); + output_w32(0x4b000000 | rm_rn_rd(rs, WZR, rt)); } static void emit_sub(u_int rs1, u_int rs2, u_int rt) { - assem_debug("sub %s, %s, %s\n", regname[rt], regname[rs1], regname[rs2]); + assem_debug("sub %s,%s,%s\n", regname[rt], regname[rs1], regname[rs2]); output_w32(0x4b000000 | rm_imm6_rn_rd(rs2, 0, rs1, rt)); } -static void emit_subs(u_int rs1,u_int rs2,u_int rt) +static void emit_movz(u_int imm, u_int rt) { - assem_debug("subs %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + assem_debug("movz %s,#%#x\n", regname[rt], imm); + output_w32(0x52800000 | imm16_rd(imm, rt)); +} + +static void emit_movz_lsl16(u_int imm, u_int rt) +{ + assem_debug("movz %s,#%#x,lsl #16\n", regname[rt], imm); + output_w32(0x52a00000 | imm16_rd(imm, rt)); +} + +static void emit_movn(u_int imm, u_int rt) +{ + assem_debug("movn %s,#%#x\n", regname[rt], imm); + output_w32(0x12800000 | imm16_rd(imm, rt)); +} + +static void emit_movn_lsl16(u_int imm,u_int rt) +{ + assem_debug("movn %s,#%#x,lsl #16\n", regname[rt], imm); + output_w32(0x12a00000 | imm16_rd(imm, rt)); +} + +static void emit_movk(u_int imm,u_int rt) +{ + assem_debug("movk %s,#%#x\n", regname[rt], imm); + output_w32(0x72800000 | imm16_rd(imm, rt)); +} + +static void emit_movk_lsl16(u_int imm,u_int rt) +{ + assert(imm<65536); + assem_debug("movk %s, #%#x, lsl #16\n", regname[rt], imm); + output_w32(0x72a00000 | imm16_rd(imm, rt)); } static void emit_zeroreg(u_int rt) { - assem_debug("mov %s,#0\n",regname[rt]); - assert(0); + emit_movz(0, rt); } static void emit_movimm(u_int imm, u_int rt) { - assem_debug("mov %s,#%#x\n", regname[rt], imm); - if ((imm & 0xffff0000) == 0) - output_w32(0x52800000 | imm16_rd(imm, rt)); - else if ((imm & 0xffff0000) == 0xffff0000) - assert(0); + if (imm < 65536) + emit_movz(imm, rt); + else if ((~imm) < 65536) + emit_movn(~imm, rt); + else if ((imm&0xffff) == 0) + emit_movz_lsl16(imm >> 16, rt); + else if (((~imm)&0xffff) == 0) + emit_movn_lsl16(~imm >> 16, rt); + else if (is_rotated_mask(imm)) { + u_int immr, imms; + gen_logical_imm(imm, &immr, &imms); + assem_debug("orr %s,wzr,#%#x\n", regname[rt], imm); + output_w32(0x32000000 | n_immr_imms_rn_rd(0, immr, imms, WZR, rt)); + } else { - output_w32(0x52800000 | imm16_rd(imm & 0xffff, rt)); - output_w32(0x72a00000 | imm16_rd(imm >> 16, rt)); + emit_movz(imm & 0xffff, rt); + emit_movk_lsl16(imm >> 16, rt); } } @@ -281,8 +466,20 @@ static void emit_readword(void *addr, u_int rt) assert(0); } +static void emit_readdword(void *addr, u_int rt) +{ + uintptr_t offset = (u_char *)addr - (u_char *)&dynarec_local; + if (!(offset & 7) && offset <= 32760) { + assem_debug("ldr %s,[x%d+%#lx]\n", regname64[rt], FP, offset); + output_w32(0xf9400000 | imm12_rn_rd(offset >> 3, FP, rt)); + } + else + assert(0); +} + static void emit_loadreg(u_int r, u_int hr) { + int is64 = 0; assert(r < 64); if (r == 0) emit_zeroreg(hr); @@ -293,10 +490,13 @@ static void emit_loadreg(u_int r, u_int hr) //case LOREG: addr = &lo; break; case CCREG: addr = &cycle_count; break; case CSREG: addr = &Status; break; - case INVCP: addr = &invc_ptr; break; + case INVCP: addr = &invc_ptr; is64 = 1; break; default: assert(r < 34); break; } - emit_readword(addr, hr); + if (is64) + emit_readdword(addr, hr); + else + emit_readword(addr, hr); } } @@ -311,6 +511,17 @@ static void emit_writeword(u_int rt, void *addr) assert(0); } +static void emit_writedword(u_int rt, void *addr) +{ + uintptr_t offset = (u_char *)addr - (u_char *)&dynarec_local; + if (!(offset & 7) && offset <= 32760) { + assem_debug("str %s,[x%d+%#lx]\n", regname64[rt], FP, offset); + output_w32(0xf9000000 | imm12_rn_rd(offset >> 2, FP, rt)); + } + else + assert(0); +} + static void emit_storereg(u_int r, u_int hr) { assert(r < 64); @@ -326,93 +537,113 @@ static void emit_storereg(u_int r, u_int hr) static void emit_test(u_int rs, u_int rt) { - assem_debug("tst %s,%s\n",regname[rs],regname[rt]); - assert(0); + assem_debug("tst %s,%s\n", regname[rs], regname[rt]); + output_w32(0x6a000000 | rm_rn_rd(rt, rs, WZR)); } -static void emit_testimm(u_int rs,int imm) +static void emit_testimm(u_int rs, u_int imm) { + u_int immr, imms; assem_debug("tst %s,#%#x\n", regname[rs], imm); - assert(0); + assert(is_rotated_mask(imm)); // good enough for PCSX + gen_logical_imm(imm, &immr, &imms); + output_w32(0xb9000000 | n_immr_imms_rn_rd(0, immr, imms, rs, WZR)); } static void emit_testeqimm(u_int rs,int imm) { assem_debug("tsteq %s,$%d\n",regname[rs],imm); - assert(0); + assert(0); // TODO eliminate emit_testeqimm } static void emit_not(u_int rs,u_int rt) { assem_debug("mvn %s,%s\n",regname[rt],regname[rs]); - assert(0); + output_w32(0x2a200000 | rm_rn_rd(rs, WZR, rt)); } static void emit_mvnmi(u_int rs,u_int rt) { assem_debug("mvnmi %s,%s\n",regname[rt],regname[rs]); - assert(0); + assert(0); // eliminate } static void emit_and(u_int rs1,u_int rs2,u_int rt) { assem_debug("and %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x0a000000 | rm_rn_rd(rs2, rs1, rt)); } static void emit_or(u_int rs1,u_int rs2,u_int rt) { assem_debug("orr %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x2a000000 | rm_rn_rd(rs2, rs1, rt)); } static void emit_orrshl_imm(u_int rs,u_int imm,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(imm < 32); assem_debug("orr %s,%s,%s,lsl #%d\n",regname[rt],regname[rt],regname[rs],imm); - assert(0); + output_w32(0x2a000000 | rm_imm6_rn_rd(rs, imm, rt, rt)); } static void emit_orrshr_imm(u_int rs,u_int imm,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(imm < 32); assem_debug("orr %s,%s,%s,lsr #%d\n",regname[rt],regname[rt],regname[rs],imm); - assert(0); -} - -static void emit_or_and_set_flags(u_int rs1,u_int rs2,u_int rt) -{ - assem_debug("orrs %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x2a400000 | rm_imm6_rn_rd(rs, imm, rt, rt)); } static void emit_xor(u_int rs1,u_int rs2,u_int rt) { assem_debug("eor %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x4a000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_addimm(u_int rs, uintptr_t imm, u_int rt) +static void emit_addimm_s(u_int s, u_int is64, u_int rs, uintptr_t imm, u_int rt) { + unused const char *st = s ? "s" : ""; + s = s ? 0x20000000 : 0; + is64 = is64 ? 0x80000000 : 0; if (imm < 4096) { - assem_debug("add %s,%s,%#lx\n", regname[rt], regname[rs], imm); - output_w32(0x11000000 | imm12_rn_rd(imm, rs, rt)); + assem_debug("add%s %s,%s,%#lx\n", st, regname[rt], regname[rs], imm); + output_w32(0x11000000 | is64 | s | imm12_rn_rd(imm, rs, rt)); } else if (-imm < 4096) { - assem_debug("sub %s,%s,%#lx\n", regname[rt], regname[rs], imm); - output_w32(0x51000000 | imm12_rn_rd(imm, rs, rt)); + assem_debug("sub%s %s,%s,%#lx\n", st, regname[rt], regname[rs], imm); + output_w32(0x51000000 | is64 | s | imm12_rn_rd(-imm, rs, rt)); + } + else if (imm < 16777216) { + assem_debug("add %s,%s,#%#lx\n",regname[rt],regname[rt],imm&0xfff000); + output_w32(0x11400000 | is64 | imm12_rn_rd(imm >> 12, rs, rt)); + if ((imm & 0xfff) || s) { + assem_debug("add%s %s,%s,#%#lx\n",st,regname[rt],regname[rs],imm&0xfff); + output_w32(0x11000000 | is64 | s | imm12_rn_rd(imm, rt, rt)); + } + } + else if (-imm < 16777216) { + assem_debug("sub %s,%s,#%#lx\n",regname[rt],regname[rt],-imm&0xfff000); + output_w32(0x51400000 | is64 | imm12_rn_rd(-imm >> 12, rs, rt)); + if ((imm & 0xfff) || s) { + assem_debug("sub%s %s,%s,#%#lx\n",st,regname[rt],regname[rs],-imm&0xfff); + output_w32(0x51000000 | is64 | s | imm12_rn_rd(-imm & 0xfff, rt, rt)); + } } else assert(0); } +static void emit_addimm(u_int rs, uintptr_t imm, u_int rt) +{ + emit_addimm_s(0, 0, rs, imm, rt); +} + +static void emit_addimm64(u_int rs, uintptr_t imm, u_int rt) +{ + emit_addimm_s(0, 1, rs, imm, rt); +} + static void emit_addimm_and_set_flags(int imm, u_int rt) { - assert(0); + emit_addimm_s(1, 0, rt, imm, rt); } static void emit_addimm_no_flags(u_int imm,u_int rt) @@ -420,181 +651,195 @@ static void emit_addimm_no_flags(u_int imm,u_int rt) emit_addimm(rt,imm,rt); } -static void emit_adcimm(u_int rs,int imm,u_int rt) +static void emit_logicop_imm(u_int op, u_int rs, u_int imm, u_int rt) { - assem_debug("adc %s,%s,#%#x\n",regname[rt],regname[rs],imm); - assert(0); + const char *names[] = { "and", "orr", "eor", "ands" }; + const char *name = names[op]; + u_int immr, imms; + op = op << 29; + if (is_rotated_mask(imm)) { + gen_logical_imm(imm, &immr, &imms); + assem_debug("%s %s,%s,#%#x\n", name, regname[rt], regname[rs], imm); + output_w32(op | 0x12000000 | n_immr_imms_rn_rd(0, immr, imms, rs, rt)); + } + else { + if (rs == HOST_TEMPREG || rt != HOST_TEMPREG) + host_tempreg_acquire(); + emit_movimm(imm, HOST_TEMPREG); + assem_debug("%s %s,%s,%s\n", name, regname[rt], regname[rs], regname[HOST_TEMPREG]); + output_w32(op | 0x0a000000 | rm_rn_rd(HOST_TEMPREG, rs, rt)); + if (rs == HOST_TEMPREG || rt != HOST_TEMPREG) + host_tempreg_release(); + } + (void)name; } -static void emit_rscimm(u_int rs,int imm,u_int rt) +static void emit_andimm(u_int rs, u_int imm, u_int rt) { - assem_debug("rsc %s,%s,#%#x\n",regname[rt],regname[rs],imm); - assert(0); + if (imm == 0) + emit_zeroreg(rt); + else + emit_logicop_imm(0, rs, imm, rt); } -static void emit_addimm64_32(u_int rsh,u_int rsl,int imm,u_int rth,u_int rtl) +static void emit_orimm(u_int rs, u_int imm, u_int rt) { - assert(0); + if (imm == 0) { + if (rs != rt) + emit_mov(rs, rt); + } + else + emit_logicop_imm(1, rs, imm, rt); } -static void emit_andimm(u_int rs,int imm,u_int rt) +static void emit_xorimm(u_int rs, u_int imm, u_int rt) { - assert(0); + if (imm == 0) { + if (rs != rt) + emit_mov(rs, rt); + } + else + emit_logicop_imm(2, rs, imm, rt); } -static void emit_orimm(u_int rs,int imm,u_int rt) +static void emit_sbfm(u_int rs,u_int imm,u_int rt) { - assert(0); + assem_debug("sbfm %s,%s,#0,#%d\n",regname[rt],regname[rs],imm); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, 0, imm, rs, rt)); } -static void emit_xorimm(u_int rs,int imm,u_int rt) +static void emit_ubfm(u_int rs,u_int imm,u_int rt) { - assert(0); + assem_debug("ubfm %s,%s,#0,#%d\n",regname[rt],regname[rs],imm); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, 0, imm, rs, rt)); } static void emit_shlimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("lsl %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); -} - -static void emit_lsls_imm(u_int rs,int imm,u_int rt) -{ - assert(imm>0); - assert(imm<32); - assem_debug("lsls %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, (31-imm)+1, 31-imm, rs, rt)); } static unused void emit_lslpls_imm(u_int rs,int imm,u_int rt) { - assert(imm>0); - assert(imm<32); - assem_debug("lslpls %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + assert(0); // eliminate } static void emit_shrimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("lsr %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, imm, 31, rs, rt)); } static void emit_sarimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("asr %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, imm, 31, rs, rt)); } static void emit_rorimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); - assem_debug("ror %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + assem_debug("ror %s,%s,#%d",regname[rt],regname[rs],imm); + output_w32(0x13800000 | rm_imm6_rn_rd(rs, imm, rs, rt)); } static void emit_signextend16(u_int rs, u_int rt) { assem_debug("sxth %s,%s\n", regname[rt], regname[rs]); - assert(0); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, 0, 15, rs, rt)); } -static void emit_shl(u_int rs,u_int shift,u_int rt) +static void emit_shl(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift < 16); - assert(0); + assem_debug("lsl %s,%s,%s",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02000 | rm_rn_rd(rshift, rs, rt)); } -static void emit_shr(u_int rs,u_int shift,u_int rt) +static void emit_shr(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("lsr %s,%s,%s\n",regname[rt],regname[rs],regname[shift]); - assert(0); -} - -static void emit_sar(u_int rs,u_int shift,u_int rt) -{ - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("asr %s,%s,%s\n",regname[rt],regname[rs],regname[shift]); - assert(0); + assem_debug("lsr %s,%s,%s\n",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02400 | rm_rn_rd(rshift, rs, rt)); } -static void emit_orrshl(u_int rs,u_int shift,u_int rt) +static void emit_sar(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("orr %s,%s,%s,lsl %s\n",regname[rt],regname[rt],regname[rs],regname[shift]); - assert(0); + assem_debug("asr %s,%s,%s\n",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02800 | rm_rn_rd(rshift, rs, rt)); } -static void emit_orrshr(u_int rs,u_int shift,u_int rt) +static void emit_cmpimm(u_int rs, u_int imm) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("orr %s,%s,%s,lsr %s\n",regname[rt],regname[rt],regname[rs],regname[shift]); - assert(0); + if (imm < 4096) { + assem_debug("cmp %s,%#x\n", regname[rs], imm); + output_w32(0x71000000 | imm12_rn_rd(imm, rs, WZR)); + } + else if (-imm < 4096) { + assem_debug("cmn %s,%#x\n", regname[rs], imm); + output_w32(0x31000000 | imm12_rn_rd(-imm, rs, WZR)); + } + else if (imm < 16777216 && !(imm & 0xfff)) { + assem_debug("cmp %s,#%#x,lsl #12\n", regname[rs], imm >> 12); + output_w32(0x71400000 | imm12_rn_rd(imm >> 12, rs, WZR)); + } + else { + host_tempreg_acquire(); + emit_movimm(imm, HOST_TEMPREG); + assem_debug("cmp %s,%s\n", regname[rs], regname[HOST_TEMPREG]); + output_w32(0x6b000000 | rm_rn_rd(HOST_TEMPREG, rs, WZR)); + host_tempreg_release(); + } } -static void emit_cmpimm(u_int rs,int imm) +static void emit_cmov_imm(u_int cond0, u_int cond1, u_int imm, u_int rt) { - assert(0); + assert(imm == 0 || imm == 1); + assert(cond0 < 0x10); + assert(cond1 < 0x10); + if (imm) { + assem_debug("csinc %s,%s,%s,%s\n",regname[rt],regname[rt],regname[WZR],condname[cond1]); + output_w32(0x1a800400 | (cond1 << 12) | rm_rn_rd(WZR, rt, rt)); + } else { + assem_debug("csel %s,%s,%s,%s\n",regname[rt],regname[WZR],regname[rt],condname[cond0]); + output_w32(0x1a800000 | (cond0 << 12) | rm_rn_rd(rt, WZR, rt)); + } } -static void emit_cmovne_imm(int imm,u_int rt) +static void emit_cmovne_imm(u_int imm,u_int rt) { - assem_debug("movne %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_NE, COND_EQ, imm, rt); } -static void emit_cmovl_imm(int imm,u_int rt) +static void emit_cmovl_imm(u_int imm,u_int rt) { - assem_debug("movlt %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_LT, COND_GE, imm, rt); } static void emit_cmovb_imm(int imm,u_int rt) { - assem_debug("movcc %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_CC, COND_CS, imm, rt); } static void emit_cmovs_imm(int imm,u_int rt) { - assem_debug("movmi %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_MI, COND_PL, imm, rt); } static void emit_cmovne_reg(u_int rs,u_int rt) { - assem_debug("movne %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,ne\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_NE << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_cmovl_reg(u_int rs,u_int rt) { - assem_debug("movlt %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,lt\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_LT << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_cmovs_reg(u_int rs,u_int rt) { - assem_debug("movmi %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,mi\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_MI << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_slti32(u_int rs,int imm,u_int rt) @@ -616,7 +861,7 @@ static void emit_sltiu32(u_int rs,int imm,u_int rt) static void emit_cmp(u_int rs,u_int rt) { assem_debug("cmp %s,%s\n",regname[rs],regname[rt]); - assert(0); + output_w32(0x6b000000 | rm_rn_rd(rt, rs, WZR)); } static void emit_set_gz32(u_int rs, u_int rt) @@ -630,7 +875,9 @@ static void emit_set_gz32(u_int rs, u_int rt) static void emit_set_nz32(u_int rs, u_int rt) { //assem_debug("set_nz32\n"); - assert(0); + if(rs!=rt) emit_mov(rs,rt); + emit_test(rs,rs); + emit_cmovne_imm(1,rt); } static void emit_set_if_less32(u_int rs1, u_int rs2, u_int rt) @@ -651,10 +898,10 @@ static void emit_set_if_carry32(u_int rs1, u_int rs2, u_int rt) emit_cmovb_imm(1,rt); } -static void emit_call(const void *a_) +static void emit_call(const void *a) { - intptr_t diff = (u_char *)a_ - out; - assem_debug("bl %p (%p+%lx)%s\n", a_, out, diff, func_name(a)); + intptr_t diff = (u_char *)a - out; + assem_debug("bl %p (%p+%lx)%s\n", a, out, diff, func_name(a)); assert(!(diff & 3)); if (-134217728 <= diff && diff <= 134217727) output_w32(0x94000000 | ((diff >> 2) & 0x03ffffff)); @@ -662,85 +909,85 @@ static void emit_call(const void *a_) assert(0); } -#pragma GCC diagnostic ignored "-Wunused-variable" -static void emit_jmp(const void *a_) +static void emit_jmp(const void *a) { - uintptr_t a = (uintptr_t)a_; - assem_debug("b %p (%p+%lx)%s\n", a_, out, (u_char *)a_ - out, func_name(a)); - assert(0); + assem_debug("b %p (%p+%lx)%s\n", a, out, (u_char *)a - out, func_name(a)); + u_int offset = genjmp(a); + output_w32(0x14000000 | offset); } -static void emit_jne(const void *a_) +static void emit_jne(const void *a) { - uintptr_t a = (uintptr_t)a_; - assem_debug("bne %p\n", a_); - assert(0); + assem_debug("bne %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_NE); } static void emit_jeq(const void *a) { - assem_debug("beq %p\n",a); - assert(0); + assem_debug("beq %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_EQ); } static void emit_js(const void *a) { - assem_debug("bmi %p\n",a); - assert(0); + assem_debug("bmi %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_MI); } static void emit_jns(const void *a) { - assem_debug("bpl %p\n",a); - assert(0); + assem_debug("bpl %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_PL); } static void emit_jl(const void *a) { - assem_debug("blt %p\n",a); - assert(0); + assem_debug("blt %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_LT); } static void emit_jge(const void *a) { - assem_debug("bge %p\n",a); - assert(0); + assem_debug("bge %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_GE); } static void emit_jno(const void *a) { - assem_debug("bvc %p\n",a); - assert(0); + assem_debug("bvc %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_VC); } static void emit_jc(const void *a) { - assem_debug("bcs %p\n",a); - assert(0); + assem_debug("bcs %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_CS); } static void emit_jcc(const void *a) { assem_debug("bcc %p\n", a); - assert(0); -} - -static void emit_callreg(u_int r) -{ - assert(r < 31); - assem_debug("blx %s\n", regname[r]); - assert(0); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_CC); } static void emit_jmpreg(u_int r) { - assem_debug("mov pc,%s\n",regname[r]); - assert(0); + assem_debug("br %s", regname64[r]); + output_w32(0xd61f0000 | rm_rn_rd(0, r, 0)); } static void emit_retreg(u_int r) { - assem_debug("ret %s\n", r == LR ? "" : regname[r]); + assem_debug("ret %s\n", r == LR ? "" : regname64[r]); output_w32(0xd65f0000 | rm_rn_rd(0, r, 0)); } @@ -749,34 +996,101 @@ static void emit_ret(void) emit_retreg(LR); } +static void emit_adr(void *addr, u_int rt) +{ + intptr_t offset = (u_char *)addr - out; + assert(-1048576 <= offset && offset < 1048576); + assem_debug("adr x%d,#%#lx\n", rt, offset); + output_w32(0x10000000 | ((offset&0x3) << 29) | (((offset>>2)&0x7ffff) << 5) | rt); +} + static void emit_readword_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldr %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldur %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0xb8400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); +} + +static void emit_strb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("strb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_strh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("strh %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_str_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("str %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0xb8204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_readdword_dualindexedx8(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldr %s, [%s,%s, uxtw #3]\n",regname64[rt],regname64[rs1],regname[rs2]); + output_w32(0xf8605800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38604800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrsb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrsb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38a04800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrh %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78604800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrsh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrsh %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78a04800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldr_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldr %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0xb8604800 | rm_rn_rd(rs2, rs1, rt)); } static void emit_movsbl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrsb %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldursb %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x38c00000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movswl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrsh %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldursh %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x78c00000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movzbl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrb %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldurb %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x38400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movzwl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrh %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldurh %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x78400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_writeword_indexed(u_int rt, int offset, u_int rs) @@ -832,41 +1146,15 @@ static void emit_clz(u_int rs,u_int rt) assert(0); } -// Load 2 immediates optimizing for small code size -static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2) -{ - assert(0); -} - -// Conditionally select one of two immediates, optimizing for small code size -// This will only be called if HAVE_CMOV_IMM is defined -static void emit_cmov2imm_e_ne_compact(int imm1,int imm2,u_int rt) -{ - assert(0); -} - // special case for checking invalid_code -static void emit_cmpmem_indexedsr12_reg(int base,u_int r,int imm) -{ - assert(imm<128&&imm>=0); - assert(r>=0&&r<16); - assem_debug("ldrb lr,%s,%s lsr #12\n",regname[base],regname[r]); - assert(0); -} - -// Used to preload hash table entries -static unused void emit_prefetchreg(u_int r) +static void emit_cmpmem_indexedsr12_reg(u_int rbase, u_int r, u_int imm) { - assem_debug("pld %s\n",regname[r]); - assert(0); -} - -// Special case for mini_ht -static void emit_ldreq_indexed(u_int rs, u_int offset, u_int rt) -{ - assert(offset<4096); - assem_debug("ldreq %s,[%s, #%#x]\n",regname[rt],regname[rs],offset); - assert(0); + host_tempreg_acquire(); + emit_shrimm(r, 12, HOST_TEMPREG); + assem_debug("ldrb %s,[%s,%s]",regname[HOST_TEMPREG],regname64[rbase],regname64[HOST_TEMPREG]); + output_w32(0x38606800 | rm_rn_rd(HOST_TEMPREG, rbase, HOST_TEMPREG)); + emit_cmpimm(HOST_TEMPREG, imm); + host_tempreg_release(); } static void emit_orrne_imm(u_int rs,int imm,u_int rt) @@ -887,26 +1175,30 @@ static unused void emit_addpl_imm(u_int rs,int imm,u_int rt) assert(0); } +static void emit_loadlp_ofs(u_int ofs, u_int rt) +{ + output_w32(0x58000000 | imm19_rt(ofs, rt)); +} + static void emit_ldst(int is_st, int is64, u_int rt, u_int rn, u_int ofs) { u_int op = 0xb9000000; - const char *ldst = is_st ? "st" : "ld"; - char rp = is64 ? 'x' : 'w'; + unused const char *ldst = is_st ? "st" : "ld"; + unused char rp = is64 ? 'x' : 'w'; assem_debug("%sr %c%d,[x%d,#%#x]\n", ldst, rp, rt, rn, ofs); is64 = is64 ? 1 : 0; assert((ofs & ((1 << (2+is64)) - 1)) == 0); ofs = (ofs >> (2+is64)); - assert(ofs <= 0xfff); if (!is_st) op |= 0x00400000; if (is64) op |= 0x40000000; - output_w32(op | (ofs << 15) | imm12_rn_rd(ofs, rn, rt)); + output_w32(op | imm12_rn_rd(ofs, rn, rt)); } static void emit_ldstp(int is_st, int is64, u_int rt1, u_int rt2, u_int rn, int ofs) { u_int op = 0x29000000; - const char *ldst = is_st ? "st" : "ld"; - char rp = is64 ? 'x' : 'w'; + unused const char *ldst = is_st ? "st" : "ld"; + unused char rp = is64 ? 'x' : 'w'; assem_debug("%sp %c%d,%c%d,[x%d,#%#x]\n", ldst, rp, rt1, rp, rt2, rn, ofs); is64 = is64 ? 1 : 0; assert((ofs & ((1 << (2+is64)) - 1)) == 0); @@ -915,7 +1207,7 @@ static void emit_ldstp(int is_st, int is64, u_int rt1, u_int rt2, u_int rn, int ofs &= 0x7f; if (!is_st) op |= 0x00400000; if (is64) op |= 0x80000000; - output_w32(op | (ofs << 15) | rm_rn_rd(rt2, rn, rt1)); + output_w32(op | imm7_rt2_rn_rt(ofs, rt2, rn, rt1)); } static void save_load_regs_all(int is_store, u_int reglist) @@ -963,57 +1255,323 @@ static void literal_pool_jumpover(int n) { } -static void emit_extjump2(u_char *addr, int target, void *linker) +// parsed by get_pointer, find_extjump_insn +static void emit_extjump2(u_char *addr, u_int target, void *linker) { - assert(0); -} + assert(((addr[3]&0xfc)==0x14) || ((addr[3]&0xff)==0x54)); // b or b.cond -static void emit_extjump(void *addr, int target) -{ - emit_extjump2(addr, target, dyna_linker); + emit_movz(target & 0xffff, 0); + emit_movk_lsl16(target >> 16, 0); + + // addr is in the current recompiled block (max 256k) + // offset shouldn't exceed +/-1MB + emit_adr(addr, 1); + emit_jmp(linker); } -static void emit_extjump_ds(void *addr, int target) +static void check_extjump2(void *src) { - emit_extjump2(addr, target, dyna_linker_ds); + u_int *ptr = src; + assert((ptr[0] & 0xffe0001f) == 0x52800000); // movz r0, #val + (void)ptr; } // put rt_val into rt, potentially making use of rs with value rs_val -static void emit_movimm_from(u_int rs_val, u_int rs, uintptr_t rt_val, u_int rt) +static void emit_movimm_from(u_int rs_val, u_int rs, u_int rt_val, u_int rt) { - intptr_t diff = rt_val - rs_val; - if (-4096 < diff && diff < 4096) + int diff = rt_val - rs_val; + if ((-4096 <= diff && diff < 4096) + || (-16777216 <= diff && diff < 16777216 && !(diff & 0xfff))) emit_addimm(rs, diff, rt); + else if (is_rotated_mask(rs_val ^ rt_val)) + emit_xorimm(rs, rs_val ^ rt_val, rt); else - // TODO: for inline_writestub, etc - assert(0); + emit_movimm(rt_val, rt); } -// return 1 if above function can do it's job cheaply +// return 1 if the above function can do it's job cheaply static int is_similar_value(u_int v1, u_int v2) { int diff = v1 - v2; - return -4096 < diff && diff < 4096; + return (-4096 <= diff && diff < 4096) + || (-16777216 <= diff && diff < 16777216 && !(diff & 0xfff)) + || is_rotated_mask(v1 ^ v2); +} + +// trashes r2 +static void pass_args64(u_int a0, u_int a1) +{ + if(a0==1&&a1==0) { + // must swap + emit_mov64(a0,2); emit_mov64(a1,1); emit_mov64(2,0); + } + else if(a0!=0&&a1==0) { + emit_mov64(a1,1); + if (a0>=0) emit_mov64(a0,0); + } + else { + if(a0>=0&&a0!=0) emit_mov64(a0,0); + if(a1>=0&&a1!=1) emit_mov64(a1,1); + } } -//#include "pcsxmem.h" +static void loadstore_extend(enum stub_type type, u_int rs, u_int rt) +{ + switch(type) { + case LOADB_STUB: emit_sbfm(rs, 7, rt); break; + case LOADBU_STUB: + case STOREB_STUB: emit_ubfm(rs, 7, rt); break; + case LOADH_STUB: emit_sbfm(rs, 15, rt); break; + case LOADHU_STUB: + case STOREH_STUB: emit_ubfm(rs, 15, rt); break; + case LOADW_STUB: + case STOREW_STUB: if (rs != rt) emit_mov(rs, rt); break; + default: assert(0); + } +} + +#include "pcsxmem.h" //#include "pcsxmem_inline.c" static void do_readstub(int n) { assem_debug("do_readstub %x\n",start+stubs[n].a*4); - assert(0); + set_jump_target(stubs[n].addr, out); + enum stub_type type = stubs[n].type; + int i = stubs[n].a; + int rs = stubs[n].b; + const struct regstat *i_regs = (void *)stubs[n].c; + u_int reglist = stubs[n].e; + const signed char *i_regmap = i_regs->regmap; + int rt; + if(itype[i]==C1LS||itype[i]==C2LS||itype[i]==LOADLR) { + rt=get_reg(i_regmap,FTEMP); + }else{ + rt=get_reg(i_regmap,rt1[i]); + } + assert(rs>=0); + int r,temp=-1,temp2=HOST_TEMPREG,regs_saved=0; + void *restore_jump = NULL, *handler_jump = NULL; + reglist|=(1<=0&&rt1[i]!=0) + reglist&=~(1<=0&&rt1[i]!=0)) { + switch(type) { + case LOADB_STUB: emit_ldrsb_dualindexed(temp2,rs,rt); break; + case LOADBU_STUB: emit_ldrb_dualindexed(temp2,rs,rt); break; + case LOADH_STUB: emit_ldrsh_dualindexed(temp2,rs,rt); break; + case LOADHU_STUB: emit_ldrh_dualindexed(temp2,rs,rt); break; + case LOADW_STUB: emit_ldr_dualindexed(temp2,rs,rt); break; + default: assert(0); + } + } + if(regs_saved) { + restore_jump=out; + emit_jmp(0); // jump to reg restore + } + else + emit_jmp(stubs[n].retaddr); // return address + set_jump_target(handler_jump, out); + + if(!regs_saved) + save_regs(reglist); + void *handler=NULL; + if(type==LOADB_STUB||type==LOADBU_STUB) + handler=jump_handler_read8; + if(type==LOADH_STUB||type==LOADHU_STUB) + handler=jump_handler_read16; + if(type==LOADW_STUB) + handler=jump_handler_read32; + assert(handler); + pass_args64(rs,temp2); + int cc=get_reg(i_regmap,CCREG); + if(cc<0) + emit_loadreg(CCREG,2); + emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2); + emit_call(handler); + // (no cycle reload after read) + if(itype[i]==C1LS||itype[i]==C2LS||(rt>=0&&rt1[i]!=0)) { + loadstore_extend(type,0,rt); + } + if(restore_jump) + set_jump_target(restore_jump, out); + restore_regs(reglist); + emit_jmp(stubs[n].retaddr); } static void inline_readstub(enum stub_type type, int i, u_int addr, signed char regmap[], int target, int adj, u_int reglist) { - assert(0); + int rs=get_reg(regmap,target); + int rt=get_reg(regmap,target); + if(rs<0) rs=get_reg(regmap,-1); + assert(rs>=0); + u_int is_dynamic=0; + uintptr_t host_addr = 0; + void *handler; + int cc=get_reg(regmap,CCREG); + //if(pcsx_direct_read(type,addr,CLOCK_ADJUST(adj+1),cc,target?rs:-1,rt)) + // return; + handler = get_direct_memhandler(mem_rtab, addr, type, &host_addr); + if (handler == NULL) { + if(rt<0||rt1[i]==0) + return; + if (addr != host_addr) { + if (host_addr >= 0x100000000ull) + abort(); // ROREG not implemented + emit_movimm_from(addr, rs, host_addr, rs); + } + switch(type) { + case LOADB_STUB: emit_movsbl_indexed(0,rs,rt); break; + case LOADBU_STUB: emit_movzbl_indexed(0,rs,rt); break; + case LOADH_STUB: emit_movswl_indexed(0,rs,rt); break; + case LOADHU_STUB: emit_movzwl_indexed(0,rs,rt); break; + case LOADW_STUB: emit_readword_indexed(0,rs,rt); break; + default: assert(0); + } + return; + } + is_dynamic=pcsxmem_is_handler_dynamic(addr); + if(is_dynamic) { + if(type==LOADB_STUB||type==LOADBU_STUB) + handler=jump_handler_read8; + if(type==LOADH_STUB||type==LOADHU_STUB) + handler=jump_handler_read16; + if(type==LOADW_STUB) + handler=jump_handler_read32; + } + + // call a memhandler + if(rt>=0&&rt1[i]!=0) + reglist&=~(1<=0&&rt1[i]!=0) + loadstore_extend(type, 0, rt); + restore_regs(reglist); } static void do_writestub(int n) { assem_debug("do_writestub %x\n",start+stubs[n].a*4); - assert(0); + set_jump_target(stubs[n].addr, out); + enum stub_type type=stubs[n].type; + int i=stubs[n].a; + int rs=stubs[n].b; + struct regstat *i_regs=(struct regstat *)stubs[n].c; + u_int reglist=stubs[n].e; + signed char *i_regmap=i_regs->regmap; + int rt,r; + if(itype[i]==C1LS||itype[i]==C2LS) { + rt=get_reg(i_regmap,r=FTEMP); + }else{ + rt=get_reg(i_regmap,r=rs2[i]); + } + assert(rs>=0); + assert(rt>=0); + int rtmp,temp=-1,temp2,regs_saved=0; + void *restore_jump = NULL, *handler_jump = NULL; + int reglist2=reglist|(1<= 0x100000000ull) + abort(); // ROREG not implemented emit_movimm_from(addr, rs, host_addr, rs); - switch(type) { + } + switch (type) { case STOREB_STUB: emit_writebyte_indexed(rt, 0, rs); break; case STOREH_STUB: emit_writehword_indexed(rt, 0, rs); break; case STOREW_STUB: emit_writeword_indexed(rt, 0, rs); break; @@ -1038,25 +1599,20 @@ static void inline_writestub(enum stub_type type, int i, u_int addr, signed char // call a memhandler save_regs(reglist); - //pass_args(rs, rt); - int cc = get_reg(regmap, CCREG); - assert(cc >= 0); - emit_addimm(cc, CLOCK_ADJUST(adj+1), 2); - //emit_movimm((uintptr_t)handler, 3); - // returns new cycle_count - - emit_readword(&last_count, HOST_TEMPREG); emit_writeword(rs, &address); // some handlers still need it - emit_add(2, HOST_TEMPREG, 2); - emit_writeword(2, &Count); - emit_mov(1, 0); + loadstore_extend(type, rt, 0); + int cc, cc_use; + cc = cc_use = get_reg(regmap, CCREG); + if (cc < 0) + emit_loadreg(CCREG, (cc_use = 2)); + emit_addimm(cc_use, CLOCK_ADJUST(adj+1), 2); + + emit_call(do_memhandler_pre); emit_call(handler); - emit_readword(&next_interupt, 0); - emit_readword(&Count, 1); - emit_writeword(0, &last_count); - emit_sub(1, 0, cc); - - emit_addimm(cc,-CLOCK_ADJUST(adj+1),cc); + emit_call(do_memhandler_post); + emit_addimm(0, -CLOCK_ADJUST(adj+1), cc_use); + if (cc < 0) + emit_storereg(CCREG, cc_use); restore_regs(reglist); } @@ -1066,24 +1622,56 @@ static void do_unalignedwritestub(int n) assert(0); } -static void do_invstub(int n) +static void set_loadlp(u_int *loadl, void *lit) { - assert(0); + uintptr_t ofs = (u_char *)lit - (u_char *)loadl; + assert((*loadl & ~0x1f) == 0x58000000); + assert((ofs & 3) == 0); + assert(ofs < 0x100000); + *loadl |= (ofs >> 2) << 5; +} + +// this output is parsed by verify_dirty, get_bounds, isclean, get_clean_addr +static void do_dirty_stub_emit_args(u_int arg0) +{ + assert(slen <= MAXBLOCK); + emit_loadlp_ofs(0, 1); // ldr x1, source + emit_loadlp_ofs(0, 2); // ldr x2, copy + emit_movz(slen*4, 3); + emit_movz(arg0 & 0xffff, 0); + emit_movk_lsl16(arg0 >> 16, 0); +} + +static void do_dirty_stub_emit_literals(u_int *loadlps) +{ + set_loadlp(&loadlps[0], out); + output_w64((uintptr_t)source); + set_loadlp(&loadlps[1], out); + output_w64((uintptr_t)copy); } -void *do_dirty_stub(int i) +static void *do_dirty_stub(int i) { assem_debug("do_dirty_stub %x\n",start+i*4); - // Careful about the code output here, verify_dirty needs to parse it. - assert(0); + u_int *loadlps = (void *)out; + do_dirty_stub_emit_args(start + i*4); + emit_call(verify_code); + void *entry = out; load_regs_entry(i); - return NULL; + if (entry == out) + entry = instr_addr[i]; + emit_jmp(instr_addr[i]); + do_dirty_stub_emit_literals(loadlps); + return entry; } static void do_dirty_stub_ds() { - // Careful about the code output here, verify_dirty needs to parse it. - assert(0); + do_dirty_stub_emit_args(start + 1); + u_int *loadlps = (void *)out; + emit_call(verify_code_ds); + emit_jmp(out + 8*2); + do_dirty_stub_emit_literals(loadlps); } /* Special assem */ @@ -1112,33 +1700,52 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) } #define multdiv_assemble multdiv_assemble_arm64 +static void do_jump_vaddr(u_int rs) +{ + if (rs != 0) + emit_mov(rs, 0); + emit_call(get_addr_ht); + emit_jmpreg(0); +} + static void do_preload_rhash(u_int r) { // Don't need this for ARM. On x86, this puts the value 0xf8 into the // register. On ARM the hash can be done with a single instruction (below) } static void do_preload_rhtbl(u_int ht) { - emit_addimm(FP, (u_char *)&mini_ht - (u_char *)&dynarec_local, ht); + emit_addimm64(FP, (u_char *)&mini_ht - (u_char *)&dynarec_local, ht); } static void do_rhash(u_int rs,u_int rh) { emit_andimm(rs, 0xf8, rh); } -static void do_miniht_load(int ht,u_int rh) { - assem_debug("ldr %s,[%s,%s]!\n",regname[rh],regname[ht],regname[rh]); - assert(0); +static void do_miniht_load(int ht, u_int rh) { + emit_add64(ht, rh, ht); + emit_ldst(0, 0, rh, ht, 0); } -static void do_miniht_jump(u_int rs,u_int rh,int ht) { - emit_cmp(rh,rs); - emit_ldreq_indexed(ht,4,15); - //emit_jmp(jump_vaddr_reg[rs]); - assert(0); +static void do_miniht_jump(u_int rs, u_int rh, u_int ht) { + emit_cmp(rh, rs); + void *jaddr = out; + emit_jeq(0); + do_jump_vaddr(rs); + + set_jump_target(jaddr, out); + assem_debug("ldr %s,[%s,#8]\n",regname64[ht], regname64[ht]); + output_w32(0xf9400000 | imm12_rn_rd(8 >> 3, ht, ht)); + emit_jmpreg(ht); } +// parsed by set_jump_target? static void do_miniht_insert(u_int return_address,u_int rt,int temp) { - assert(0); + emit_movz_lsl16((return_address>>16)&0xffff,rt); + emit_movk(return_address&0xffff,rt); + add_to_linker(out,return_address,1); + emit_adr(out,temp); + emit_writedword(temp,&mini_ht[(return_address&0xFF)>>3][1]); + emit_writeword(rt,&mini_ht[(return_address&0xFF)>>3][0]); } static void mark_clear_cache(void *target) diff --git a/libpcsxcore/new_dynarec/assem_arm64.h b/libpcsxcore/new_dynarec/assem_arm64.h index 6789f178..fe12ad75 100644 --- a/libpcsxcore/new_dynarec/assem_arm64.h +++ b/libpcsxcore/new_dynarec/assem_arm64.h @@ -3,21 +3,15 @@ #define EXCLUDE_REG -1 #define HOST_IMM8 1 -#define HAVE_CMOV_IMM 1 #define RAM_SIZE 0x200000 -//#define REG_SHIFT 2 - /* calling convention: r0 -r17: caller-save r19-r29: callee-save */ -#define ARG1_REG 0 -#define ARG2_REG 1 -#define ARG3_REG 2 -#define ARG4_REG 3 - -#define SP 30 +#define SP 31 +#define WZR SP +#define XZR SP #define LR 30 #define HOST_TEMPREG LR diff --git a/libpcsxcore/new_dynarec/linkage_arm.S b/libpcsxcore/new_dynarec/linkage_arm.S index 778a67f0..fcb4e1a7 100644 --- a/libpcsxcore/new_dynarec/linkage_arm.S +++ b/libpcsxcore/new_dynarec/linkage_arm.S @@ -476,14 +476,6 @@ FUNCTION(cc_interrupt): b .E1 .size cc_interrupt, .-cc_interrupt - .align 2 -FUNCTION(do_interrupt): - ldr r0, [fp, #LO_pcaddr] - bl get_addr_ht - add r10, r10, #2 - mov pc, r0 - .size do_interrupt, .-do_interrupt - .align 2 FUNCTION(fp_exception): mov r2, #0x10000000 diff --git a/libpcsxcore/new_dynarec/linkage_arm64.S b/libpcsxcore/new_dynarec/linkage_arm64.S index 397874c8..060ac48a 100644 --- a/libpcsxcore/new_dynarec/linkage_arm64.S +++ b/libpcsxcore/new_dynarec/linkage_arm64.S @@ -84,7 +84,7 @@ DRC_VAR(restore_candidate, 512) /* r0 = virtual target address */ /* r1 = instruction to patch */ .macro dyna_linker_main - /* XXX: should be able to do better than this... */ + /* XXX TODO: should be able to do better than this... */ bl get_addr_ht br x0 .endm @@ -113,12 +113,6 @@ FUNCTION(dyna_linker_ds): .align 2 -FUNCTION(jump_vaddr): - bl abort - .size jump_vaddr, .-jump_vaddr - - .align 2 - FUNCTION(verify_code_ds): bl abort FUNCTION(verify_code): @@ -131,14 +125,49 @@ FUNCTION(verify_code): .align 2 FUNCTION(cc_interrupt): - bl abort + ldr w0, [rFP, #LO_last_count] + mov w2, #0x1fc + add rCC, w0, rCC + str wzr, [rFP, #LO_pending_exception] + and w2, w2, rCC, lsr #17 + add x3, rFP, #LO_restore_candidate + str rCC, [rFP, #LO_cycle] /* PCSX cycles */ +# str rCC, [rFP, #LO_reg_cop0+36] /* Count */ + ldr w19, [x3, w2, uxtw] + mov x21, lr + cbnz w19, 4f +1: + bl gen_interupt + mov lr, x21 + ldr rCC, [rFP, #LO_cycle] + ldr w0, [rFP, #LO_next_interupt] + ldr w1, [rFP, #LO_pending_exception] + ldr w2, [rFP, #LO_stop] + str w0, [rFP, #LO_last_count] + sub rCC, rCC, w0 + cbnz w2, new_dyna_leave + cbnz w1, 2f + ret +2: + ldr w0, [rFP, #LO_pcaddr] + bl get_addr_ht + br x0 +4: + /* Move 'dirty' blocks to the 'clean' list */ + lsl w20, w2, #3 + str wzr, [x3, w2, uxtw] +5: + mov w0, w20 + add w20, w20, #1 + tbz w19, #0, 6f + bl clean_blocks +6: + lsr w19, w19, #1 + tst w20, #31 + bne 5b + b 1b .size cc_interrupt, .-cc_interrupt - .align 2 -FUNCTION(do_interrupt): - bl abort - .size do_interrupt, .-do_interrupt - .align 2 FUNCTION(fp_exception): mov w2, #0x10000000 @@ -239,26 +268,94 @@ FUNCTION(new_dyna_leave): .align 2 +.macro memhandler_pre + /* w0 = adddr/data, x1 = rhandler, w2 = cycles, x3 = whandler */ + ldr w4, [rFP, #LO_last_count] + add w4, w4, w2 + str w4, [rFP, #LO_cycle] +.endm + +.macro memhandler_post + ldr w2, [rFP, #LO_next_interupt] + ldr w1, [rFP, #LO_cycle] + sub w0, w1, w2 + str w2, [rFP, #LO_last_count] +.endm + +FUNCTION(do_memhandler_pre): + memhandler_pre + ret + +FUNCTION(do_memhandler_post): + memhandler_post + ret + +.macro pcsx_read_mem readop tab_shift + /* w0 = address, x1 = handler_tab, w2 = cycles */ + stp xzr, x30, [sp, #-16]! + ubfm w4, w0, #\tab_shift, #11 + ldr x3, [x1, w4, uxtw #3] + adds x3, x3, x3 + bcs 0f + \readop w0, [x3, w4, uxtw #\tab_shift] + ret +0: + memhandler_pre + blr x3 +.endm + FUNCTION(jump_handler_read8): - bl abort + add x1, x1, #0x1000/4*4 + 0x1000/2*4 /* shift to r8 part */ + pcsx_read_mem ldrb, 0 + b handler_read_end FUNCTION(jump_handler_read16): - bl abort + add x1, x1, #0x1000/4*4 /* shift to r16 part */ + pcsx_read_mem ldrh, 1 + b handler_read_end FUNCTION(jump_handler_read32): - bl abort + pcsx_read_mem ldr, 2 + +handler_read_end: + ldp xzr, x30, [sp], #16 + ret + +.macro pcsx_write_mem wrtop movop tab_shift + /* w0 = address, w1 = data, w2 = cycles, x3 = handler_tab */ + stp xzr, x30, [sp, #-16]! + ubfm w4, w0, #\tab_shift, #11 + ldr x3, [x3, w4, uxtw #3] + str w0, [rFP, #LO_address] /* some handlers still need it... */ + adds x3, x3, x3 +# str lr, [rFP, #0] + bcs 0f + mov w0, w2 /* cycle return */ + \wrtop w1, [x3, w4, uxtw #\tab_shift] + ret +0: + \movop w0, w1 + memhandler_pre + blr x3 +.endm FUNCTION(jump_handler_write8): - bl abort + add x3, x3, #0x1000/4*4 + 0x1000/2*4 /* shift to r8 part */ + pcsx_write_mem strb uxtb 0 + b handler_write_end FUNCTION(jump_handler_write16): - bl abort + add x3, x3, #0x1000/4*4 /* shift to r16 part */ + pcsx_write_mem strh uxth 1 + b handler_write_end FUNCTION(jump_handler_write32): - bl abort + pcsx_write_mem str mov 2 -FUNCTION(jump_handler_write_h): - bl abort +handler_write_end: + memhandler_post + ldp xzr, x30, [sp], #16 + ret FUNCTION(jump_handle_swl): bl abort diff --git a/libpcsxcore/new_dynarec/linkage_offsets.h b/libpcsxcore/new_dynarec/linkage_offsets.h index 24b8e66d..82d27bd4 100644 --- a/libpcsxcore/new_dynarec/linkage_offsets.h +++ b/libpcsxcore/new_dynarec/linkage_offsets.h @@ -35,7 +35,7 @@ #define LO_scratch_buf_ptr (LO_invc_ptr + PTRSZ) #define LO_align1 (LO_scratch_buf_ptr + PTRSZ) #define LO_mini_ht (LO_align1 + PTRSZ*2) -#define LO_restore_candidate (LO_mini_ht + 256) +#define LO_restore_candidate (LO_mini_ht + PTRSZ*32*2) #define LO_dynarec_local_size (LO_restore_candidate + 512) #define LO_cop2_to_scratch_buf (LO_scratch_buf_ptr - LO_reg_cop2d) diff --git a/libpcsxcore/new_dynarec/new_dynarec.c b/libpcsxcore/new_dynarec/new_dynarec.c index 7c59a7e8..08153298 100644 --- a/libpcsxcore/new_dynarec/new_dynarec.c +++ b/libpcsxcore/new_dynarec/new_dynarec.c @@ -38,6 +38,7 @@ static int sceBlock; #include "../psxhle.h" //emulator interface #include "emu_if.h" //emulator interface +#define noinline __attribute__((noinline,noclone)) #ifndef ARRAY_SIZE #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) #endif @@ -202,7 +203,7 @@ struct link_entry extern int pcaddr; extern int pending_exception; extern int branch_target; - extern u_int mini_ht[32][2]; + extern uintptr_t mini_ht[32][2]; extern u_char restore_candidate[512]; /* registers that may be allocated */ @@ -421,7 +422,7 @@ static int doesnt_expire_soon(void *tcaddr) // Get address from virtual address // This is called from the recompiled JR/JALR instructions -void *get_addr(u_int vaddr) +void noinline *get_addr(u_int vaddr) { u_int page=get_page(vaddr); u_int vpage=get_vpage(vaddr); @@ -489,7 +490,7 @@ void clear_all_regs(signed char regmap[]) for (hr=0;hr %x (%d)\n",src,vaddr,page); - int *ptr=(int *)(src+4); - assert((*ptr&0x0fff0000)==0x059f0000); - (void)ptr; + check_extjump2(src); ll_add(jump_out+page,vaddr,src); //void *ptr=get_pointer(src); //inv_debug("add_link: Pointer is to %p\n",ptr); @@ -1905,7 +1935,7 @@ static void pagespan_alloc(struct regstat *current,int i) static void add_stub(enum stub_type type, void *addr, void *retaddr, u_int a, uintptr_t b, uintptr_t c, u_int d, u_int e) { - assert(a < ARRAY_SIZE(stubs)); + assert(stubcount < ARRAY_SIZE(stubs)); stubs[stubcount].type = type; stubs[stubcount].addr = addr; stubs[stubcount].retaddr = retaddr; @@ -2380,24 +2410,29 @@ static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override) } if(type==MTYPE_8020) { // RAM 80200000+ mirror + host_tempreg_acquire(); emit_andimm(addr,~0x00e00000,HOST_TEMPREG); addr=*addr_reg_override=HOST_TEMPREG; type=0; } else if(type==MTYPE_0000) { // RAM 0 mirror + host_tempreg_acquire(); emit_orimm(addr,0x80000000,HOST_TEMPREG); addr=*addr_reg_override=HOST_TEMPREG; type=0; } else if(type==MTYPE_A000) { // RAM A mirror + host_tempreg_acquire(); emit_andimm(addr,~0x20000000,HOST_TEMPREG); addr=*addr_reg_override=HOST_TEMPREG; type=0; } else if(type==MTYPE_1F80) { // scratchpad if (psxH == (void *)0x1f800000) { + host_tempreg_acquire(); emit_addimm(addr,-0x1f800000,HOST_TEMPREG); emit_cmpimm(HOST_TEMPREG,0x1000); + host_tempreg_release(); jaddr=out; emit_jc(0); } @@ -2419,6 +2454,7 @@ static void *emit_fastpath_cmp_jump(int i,int addr,int *addr_reg_override) #endif emit_jno(0); if(ram_offset!=0) { + host_tempreg_acquire(); emit_addimm(addr,ram_offset,HOST_TEMPREG); addr=*addr_reg_override=HOST_TEMPREG; } @@ -2461,7 +2497,7 @@ static void load_assemble(int i,struct regstat *i_regs) int offset; void *jaddr=0; int memtarget=0,c=0; - int fastload_reg_override=0; + int fastio_reg_override=-1; u_int hr,reglist=0; tl=get_reg(i_regs->regmap,rt1[i]); s=get_reg(i_regs->regmap,rs1[i]); @@ -2501,12 +2537,13 @@ static void load_assemble(int i,struct regstat *i_regs) if(rs1[i]!=29||start<0x80001000||start>=0x80000000+RAM_SIZE) #endif { - jaddr=emit_fastpath_cmp_jump(i,addr,&fastload_reg_override); + jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override); } } else if(ram_offset&&memtarget) { + host_tempreg_acquire(); emit_addimm(addr,ram_offset,HOST_TEMPREG); - fastload_reg_override=HOST_TEMPREG; + fastio_reg_override=HOST_TEMPREG; } int dummy=(rt1[i]==0)||(tl!=get_reg(i_regs->regmap,rt1[i])); // ignore loads to r0 and unneeded reg if (opcode[i]==0x20) { // LB @@ -2515,7 +2552,7 @@ static void load_assemble(int i,struct regstat *i_regs) { int x=0,a=tl; if(!c) a=addr; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_movsbl_indexed(x,a,tl); } @@ -2531,7 +2568,7 @@ static void load_assemble(int i,struct regstat *i_regs) if(!dummy) { int x=0,a=tl; if(!c) a=addr; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_movswl_indexed(x,a,tl); } if(jaddr) @@ -2544,7 +2581,7 @@ static void load_assemble(int i,struct regstat *i_regs) if(!c||memtarget) { if(!dummy) { int a=addr; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_readword_indexed(0,a,tl); } if(jaddr) @@ -2558,7 +2595,7 @@ static void load_assemble(int i,struct regstat *i_regs) if(!dummy) { int x=0,a=tl; if(!c) a=addr; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_movzbl_indexed(x,a,tl); } @@ -2573,7 +2610,7 @@ static void load_assemble(int i,struct regstat *i_regs) if(!dummy) { int x=0,a=tl; if(!c) a=addr; - if(fastload_reg_override) a=fastload_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_movzwl_indexed(x,a,tl); } if(jaddr) @@ -2589,6 +2626,8 @@ static void load_assemble(int i,struct regstat *i_regs) assert(0); } } + if (fastio_reg_override == HOST_TEMPREG) + host_tempreg_release(); } #ifndef loadlr_assemble @@ -2608,7 +2647,7 @@ void store_assemble(int i,struct regstat *i_regs) enum stub_type type; int memtarget=0,c=0; int agr=AGEN1+(i&1); - int faststore_reg_override=0; + int fastio_reg_override=-1; u_int hr,reglist=0; tl=get_reg(i_regs->regmap,rs2[i]); s=get_reg(i_regs->regmap,rs1[i]); @@ -2630,18 +2669,19 @@ void store_assemble(int i,struct regstat *i_regs) if(offset||s<0||c) addr=temp; else addr=s; if(!c) { - jaddr=emit_fastpath_cmp_jump(i,addr,&faststore_reg_override); + jaddr=emit_fastpath_cmp_jump(i,addr,&fastio_reg_override); } else if(ram_offset&&memtarget) { + host_tempreg_acquire(); emit_addimm(addr,ram_offset,HOST_TEMPREG); - faststore_reg_override=HOST_TEMPREG; + fastio_reg_override=HOST_TEMPREG; } if (opcode[i]==0x28) { // SB if(!c||memtarget) { int x=0,a=temp; if(!c) a=addr; - if(faststore_reg_override) a=faststore_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_writebyte_indexed(tl,x,a); } type=STOREB_STUB; @@ -2650,7 +2690,7 @@ void store_assemble(int i,struct regstat *i_regs) if(!c||memtarget) { int x=0,a=temp; if(!c) a=addr; - if(faststore_reg_override) a=faststore_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_writehword_indexed(tl,x,a); } type=STOREH_STUB; @@ -2658,7 +2698,7 @@ void store_assemble(int i,struct regstat *i_regs) if (opcode[i]==0x2B) { // SW if(!c||memtarget) { int a=addr; - if(faststore_reg_override) a=faststore_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_writeword_indexed(tl,0,a); } type=STOREW_STUB; @@ -2667,6 +2707,8 @@ void store_assemble(int i,struct regstat *i_regs) assert(0); type=STORED_STUB; } + if(fastio_reg_override==HOST_TEMPREG) + host_tempreg_release(); if(jaddr) { // PCSX store handlers don't check invcode again reglist|=1<regmap==regs[i].regmap); // not delay slot @@ -2712,7 +2755,9 @@ void store_assemble(int i,struct regstat *i_regs) wb_dirtys(regs[i].regmap_entry,regs[i].wasdirty); emit_movimm(start+i*4+4,0); emit_writeword(0,&pcaddr); - emit_jmp(do_interrupt); + emit_addimm(HOST_CCREG,2,HOST_CCREG); + emit_call(get_addr_ht); + emit_jmpreg(0); } } } @@ -2948,7 +2993,13 @@ static void cop0_assemble(int i,struct regstat *i_regs) assert(!is_delayslot); emit_readword(&pending_exception,14); emit_test(14,14); - emit_jne(&do_interrupt); + void *jaddr = out; + emit_jeq(0); + emit_readword(&pcaddr, 0); + emit_addimm(HOST_CCREG,2,HOST_CCREG); + emit_call(get_addr_ht); + emit_jmpreg(0); + set_jump_target(jaddr, out); } emit_loadreg(rs1[i],s); } @@ -3117,7 +3168,7 @@ static void c2ls_assemble(int i,struct regstat *i_regs) void *jaddr2=NULL; enum stub_type type; int agr=AGEN1+(i&1); - int fastio_reg_override=0; + int fastio_reg_override=-1; u_int hr,reglist=0; u_int copr=(source[i]>>16)&0x1f; s=get_reg(i_regs->regmap,rs1[i]); @@ -3161,12 +3212,13 @@ static void c2ls_assemble(int i,struct regstat *i_regs) jaddr2=emit_fastpath_cmp_jump(i,ar,&fastio_reg_override); } else if(ram_offset&&memtarget) { + host_tempreg_acquire(); emit_addimm(ar,ram_offset,HOST_TEMPREG); fastio_reg_override=HOST_TEMPREG; } if (opcode[i]==0x32) { // LWC2 int a=ar; - if(fastio_reg_override) a=fastio_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_readword_indexed(0,a,tl); } if (opcode[i]==0x3a) { // SWC2 @@ -3174,10 +3226,12 @@ static void c2ls_assemble(int i,struct regstat *i_regs) if(!offset&&!c&&s>=0) emit_mov(s,ar); #endif int a=ar; - if(fastio_reg_override) a=fastio_reg_override; + if(fastio_reg_override>=0) a=fastio_reg_override; emit_writeword_indexed(tl,0,a); } } + if(fastio_reg_override==HOST_TEMPREG) + host_tempreg_release(); if(jaddr2) add_stub_r(type,jaddr2,out,i,ar,i_regs,ccadj[i],reglist); if(opcode[i]==0x3a) // SWC2 @@ -3198,7 +3252,9 @@ static void c2ls_assemble(int i,struct regstat *i_regs) #endif } if (opcode[i]==0x32) { // LWC2 + host_tempreg_acquire(); cop2_put_dreg(copr,tl,HOST_TEMPREG); + host_tempreg_release(); } } @@ -4117,6 +4173,13 @@ static void emit_extjump_ds(void *addr, u_int target) emit_extjump2(addr, target, dyna_linker_ds); } +// Load 2 immediates optimizing for small code size +static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2) +{ + emit_movimm(imm1,rt1); + emit_movimm_from(imm1,rt1,imm2,rt2); +} + void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert) { int count; @@ -4172,7 +4235,7 @@ void do_cc(int i,signed char i_regmap[],int *adj,int addr,int taken,int invert) static void do_ccstub(int n) { literal_pool(256); - assem_debug("do_ccstub %lx\n",start+stubs[n].b*4); + assem_debug("do_ccstub %x\n",start+(u_int)stubs[n].b*4); set_jump_target(stubs[n].addr, out); int i=stubs[n].b; if(stubs[n].d==NULLDS) { @@ -4371,7 +4434,10 @@ static void do_ccstub(int n) }else{ load_all_regs(branch_regs[i].regmap); } - emit_jmp(stubs[n].retaddr); + if (stubs[n].retaddr) + emit_jmp(stubs[n].retaddr); + else + do_jump_vaddr(stubs[n].e); } static void add_to_linker(void *addr, u_int target, int ext) @@ -4564,7 +4630,7 @@ static void rjump_assemble(int i,struct regstat *i_regs) //if(adj) emit_addimm(cc,2*(ccadj[i]+2-adj),cc); // ??? - Shouldn't happen //assert(adj==0); emit_addimm_and_set_flags(CLOCK_ADJUST(ccadj[i]+2),HOST_CCREG); - add_stub(CC_STUB,out,jump_vaddr_reg[rs],0,i,-1,TAKEN,0); + add_stub(CC_STUB,out,NULL,0,i,-1,TAKEN,rs); if(itype[i+1]==COP0&&(source[i+1]&0x3f)==0x10) // special case for RFE emit_jmp(0); @@ -4578,7 +4644,7 @@ static void rjump_assemble(int i,struct regstat *i_regs) else #endif { - emit_jmp(jump_vaddr_reg[rs]); + do_jump_vaddr(rs); } #ifdef CORTEX_A8_BRANCH_PREDICTION_HACK if(rt1[i]!=31&&i>16)==0x1000) literal_pool(1024); else -@@ -10256,7 +10277,7 @@ int new_recompile_block(int addr) +@@ -8767,7 +8786,7 @@ int new_recompile_block(int addr) } } // External Branch Targets (jump_in) @@ -256,7 +267,7 @@ index 6d7069d..586a6db 100644 for(i=0;i> 4) & 3; emit_readword(&rcnts[t].mode, rt); + host_tempreg_acquire(); emit_andimm(rt, ~0x1800, HOST_TEMPREG); emit_writeword(HOST_TEMPREG, &rcnts[t].mode); + host_tempreg_release(); mov_loadtype_adj(type, rt, rt); goto hit; } -- 2.39.2