X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?p=pcsx_rearmed.git;a=blobdiff_plain;f=libpcsxcore%2Fnew_dynarec%2Fassem_arm64.c;h=a0c628b5892242ff230593bf9e5f6160bd212a2c;hp=fabd7dbae72f7ba947c81c8929ac9d53ce771e76;hb=d1e4ebd9988a9a5d9fb38b89f19e24b9ab6029d7;hpb=7c3a5182da4384e21a6ace037583fae399de5a02 diff --git a/libpcsxcore/new_dynarec/assem_arm64.c b/libpcsxcore/new_dynarec/assem_arm64.c index fabd7dba..a0c628b5 100644 --- a/libpcsxcore/new_dynarec/assem_arm64.c +++ b/libpcsxcore/new_dynarec/assem_arm64.c @@ -1,7 +1,8 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Mupen64plus/PCSX - assem_arm64.c * * Copyright (C) 2009-2011 Ari64 * - * Copyright (C) 2010-2021 notaz * + * Copyright (C) 2009-2018 Gillou68310 * + * Copyright (C) 2021 notaz * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * @@ -27,33 +28,50 @@ u_char *translation_cache; #else u_char translation_cache[1 << TARGET_SIZE_2] __attribute__((aligned(4096))); #endif +static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)]; #define CALLER_SAVE_REGS 0x0007ffff #define unused __attribute__((unused)) -static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)]; - -//void indirect_jump_indexed(); -//void indirect_jump(); -void do_interrupt(); -//void jump_vaddr_r0(); - -void * const jump_vaddr_reg[32]; +void do_memhandler_pre(); +void do_memhandler_post(); /* Linker */ - -static void set_jump_target(void *addr, void *target_) +static void set_jump_target(void *addr, void *target) { - assert(0); + u_int *ptr = addr; + intptr_t offset = (u_char *)target - (u_char *)addr; + + if((*ptr&0xFC000000)==0x14000000) { + assert(offset>=-134217728LL&&offset<134217728LL); + *ptr=(*ptr&0xFC000000)|((offset>>2)&0x3ffffff); + } + else if((*ptr&0xff000000)==0x54000000) { + // Conditional branch are limited to +/- 1MB + // block max size is 256k so branching beyond the +/- 1MB limit + // should only happen when jumping to an already compiled block (see add_link) + // a workaround would be to do a trampoline jump via a stub at the end of the block + assert(offset>=-1048576LL&&offset<1048576LL); + *ptr=(*ptr&0xFF00000F)|(((offset>>2)&0x7ffff)<<5); + } + else if((*ptr&0x9f000000)==0x10000000) { //adr + // generated by do_miniht_insert + assert(offset>=-1048576LL&&offset<1048576LL); + *ptr=(*ptr&0x9F00001F)|(offset&0x3)<<29|((offset>>2)&0x7ffff)<<5; + } + else + assert(0); // should not happen } // from a pointer to external jump stub (which was produced by emit_extjump2) // find where the jumping insn is static void *find_extjump_insn(void *stub) { - assert(0); - return NULL; + int *ptr = (int *)stub + 2; + assert((*ptr&0x9f000000) == 0x10000000); // adr + int offset = (((signed int)(*ptr<<8)>>13)<<2)|((*ptr>>29)&0x3); + return ptr + offset / 4; } // find where external branch is liked to using addr of it's stub: @@ -62,9 +80,9 @@ static void *find_extjump_insn(void *stub) // return addr where that branch jumps to static void *get_pointer(void *stub) { - //printf("get_pointer(%x)\n",(int)stub); - assert(0); - return NULL; + int *i_ptr = find_extjump_insn(stub); + assert((*i_ptr&0xfc000000) == 0x14000000); // b + return (u_char *)i_ptr+(((signed int)(*i_ptr<<6)>>6)<<2); } // Find the "clean" entry point from a "dirty" entry point @@ -81,18 +99,36 @@ static int verify_dirty(u_int *ptr) return 0; } -// This doesn't necessarily find all clean entry points, just -// guarantees that it's not dirty static int isclean(void *addr) { - assert(0); - return 0; + u_int *ptr = addr; + return (*ptr >> 24) != 0x58; // the only place ldr (literal) is used +} + +static uint64_t get_from_ldr_literal(const u_int *i) +{ + signed int ofs; + assert((i[0] & 0xff000000) == 0x58000000); + ofs = i[0] << 8; + ofs >>= 5+8; + return *(uint64_t *)(i + ofs); +} + +static uint64_t get_from_movz(const u_int *i) +{ + assert((i[0] & 0x7fe00000) == 0x52800000); + return (i[0] >> 5) & 0xffff; } // get source that block at addr was compiled from (host pointers) static void get_bounds(void *addr, u_char **start, u_char **end) { - assert(0); + const u_int *ptr = addr; + assert((ptr[0] & 0xff00001f) == 0x58000001); // ldr x1, source + assert((ptr[1] & 0xff00001f) == 0x58000002); // ldr x2, copy + assert((ptr[2] & 0xffe0001f) == 0x52800003); // movz w3, #slen*4 + *start = (u_char *)get_from_ldr_literal(&ptr[0]); + *end = *start + get_from_movz(&ptr[2]); } // Allocate a specific ARM register. @@ -128,10 +164,27 @@ static void alloc_cc(struct regstat *cur,int i) /* Assembler */ static unused const char *regname[32] = { - "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", - "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" - "ip0", "ip1", "r18", "r19", "r20", "r21", "r22", "r23" - "r24", "r25", "r26", "r27", "r28", "fp", "lr", "sp" + "w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7", + "w8", "w9", "w10", "w11", "w12", "w13", "w14", "w15", + "ip0", "ip1", "w18", "w19", "w20", "w21", "w22", "w23", + "w24", "w25", "w26", "w27", "w28", "wfp", "wlr", "wsp" +}; + +static unused const char *regname64[32] = { + "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + "ip0", "ip1", "x18", "x19", "x20", "x21", "x22", "x23", + "x24", "x25", "x26", "x27", "x28", "fp", "lr", "sp" +}; + +enum { + COND_EQ, COND_NE, COND_CS, COND_CC, COND_MI, COND_PL, COND_VS, COND_VC, + COND_HI, COND_LS, COND_GE, COND_LT, COND_GT, COND_LE, COND_AW, COND_NV +}; + +static unused const char *condname[16] = { + "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", + "hi", "ls", "ge", "lt", "gt", "le", "aw", "nv" }; static void output_w32(u_int word) @@ -140,21 +193,38 @@ static void output_w32(u_int word) out += 4; } +static void output_w64(uint64_t dword) +{ + *((uint64_t *)out) = dword; + out+=8; +} + +/* static u_int rm_rd(u_int rm, u_int rd) { assert(rm < 31); assert(rd < 31); return (rm << 16) | rd; } +*/ static u_int rm_rn_rd(u_int rm, u_int rn, u_int rd) { - assert(rm < 31); - assert(rn < 31); - assert(rd < 31); + assert(rm < 32); + assert(rn < 32); + assert(rd < 32); return (rm << 16) | (rn << 5) | rd; } +static u_int imm7_rt2_rn_rt(u_int imm7, u_int rt2, u_int rn, u_int rt) +{ + assert(imm7 < 0x80); + assert(rt2 < 31); + assert(rn < 32); + assert(rt < 31); + return (imm7 << 15) | (rt2 << 10) | (rn << 5) | rt; +} + static u_int rm_imm6_rn_rd(u_int rm, u_int imm6, u_int rn, u_int rd) { assert(imm6 <= 63); @@ -171,102 +241,217 @@ static u_int imm16_rd(u_int imm16, u_int rd) static u_int imm12_rn_rd(u_int imm12, u_int rn, u_int rd) { assert(imm12 < 0x1000); + assert(rn < 32); + assert(rd < 32); + return (imm12 << 10) | (rn << 5) | rd; +} + +static u_int imm9_rn_rt(u_int imm9, u_int rn, u_int rd) +{ + assert(imm9 < 0x200); assert(rn < 31); assert(rd < 31); - return (imm12 << 10) | (rn << 5) | rd; + return (imm9 << 12) | (rn << 5) | rd; } -#pragma GCC diagnostic ignored "-Wunused-function" -static u_int genjmp(u_char *addr) +static u_int imm19_rt(u_int imm19, u_int rt) +{ + assert(imm19 < 0x80000); + assert(rt < 31); + return (imm19 << 5) | rt; +} + +static u_int n_immr_imms_rn_rd(u_int n, u_int immr, u_int imms, u_int rn, u_int rd) +{ + assert(n < 2); + assert(immr < 0x40); + assert(imms < 0x40); + assert(rn < 32); + assert(rd < 32); + return (n << 22) | (immr << 16) | (imms << 10) | (rn << 5) | rd; +} + +static u_int genjmp(const u_char *addr) { intptr_t offset = addr - out; + if ((uintptr_t)addr < 3) return 0; // a branch that will be patched later if (offset < -134217728 || offset > 134217727) { - if ((uintptr_t)addr > 2) { - SysPrintf("%s: out of range: %08x\n", __func__, offset); - exit(1); - } + SysPrintf("%s: out of range: %p %lx\n", __func__, addr, offset); + abort(); return 0; } - return ((u_int)offset >> 2) & 0x01ffffff; + return ((u_int)offset >> 2) & 0x03ffffff; } -static u_int genjmpcc(u_char *addr) +static u_int genjmpcc(const u_char *addr) { intptr_t offset = addr - out; + if ((uintptr_t)addr < 3) return 0; if (offset < -1048576 || offset > 1048572) { - if ((uintptr_t)addr > 2) { - SysPrintf("%s: out of range: %08x\n", __func__, offset); - exit(1); - } + SysPrintf("%s: out of range: %p %lx\n", __func__, addr, offset); + abort(); + return 0; + } + return ((u_int)offset >> 2) & 0x7ffff; +} + +static uint32_t is_mask(u_int value) +{ + return value && ((value + 1) & value) == 0; +} + +// This function returns true if the argument contains a +// non-empty sequence of ones (possibly rotated) with the remainder zero. +static uint32_t is_rotated_mask(u_int value) +{ + if (value == 0) return 0; + if (is_mask((value - 1) | value)) + return 1; + return is_mask((~value - 1) | ~value); +} + +static void gen_logical_imm(u_int value, u_int *immr, u_int *imms) +{ + int lzeros, tzeros, ones; + assert(value != 0); + if (is_mask((value - 1) | value)) { + lzeros = __builtin_clz(value); + tzeros = __builtin_ctz(value); + ones = 32 - lzeros - tzeros; + *immr = (32 - tzeros) & 31; + *imms = ones - 1; + return; } - return ((u_int)offset >> 2) & 0xfffff; + value = ~value; + if (is_mask((value - 1) | value)) { + lzeros = __builtin_clz(value); + tzeros = __builtin_ctz(value); + ones = 32 - lzeros - tzeros; + *immr = 31 - tzeros; + *imms = 31 - ones; + return; + } + assert(0); } static void emit_mov(u_int rs, u_int rt) { assem_debug("mov %s,%s\n", regname[rt], regname[rs]); - output_w32(0x2a0003e0 | rm_rd(rs, rt)); + output_w32(0x2a000000 | rm_rn_rd(rs, WZR, rt)); +} + +static void emit_mov64(u_int rs, u_int rt) +{ + assem_debug("mov %s,%s\n", regname64[rt], regname64[rs]); + output_w32(0xaa000000 | rm_rn_rd(rs, WZR, rt)); } static void emit_movs(u_int rs, u_int rt) { + assert(0); // misleading assem_debug("movs %s,%s\n", regname[rt], regname[rs]); output_w32(0x31000000 | imm12_rn_rd(0, rs, rt)); } static void emit_add(u_int rs1, u_int rs2, u_int rt) { - assem_debug("add %s, %s, %s\n", regname[rt], regname[rs1], regname[rs2]); - output_w32(0x0b000000 | rm_imm6_rn_rd(rs2, 0, rs1, rt)); + assem_debug("add %s,%s,%s\n", regname[rt], regname[rs1], regname[rs2]); + output_w32(0x0b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_sbc(u_int rs1,u_int rs2,u_int rt) +static void emit_add64(u_int rs1, u_int rs2, u_int rt) { - assem_debug("sbc %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + assem_debug("add %s,%s,%s\n", regname64[rt], regname64[rs1], regname64[rs2]); + output_w32(0x8b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_neg(u_int rs, u_int rt) +#pragma GCC diagnostic ignored "-Wunused-function" +static void emit_adds(u_int rs1, u_int rs2, u_int rt) { - assem_debug("rsb %s,%s,#0\n",regname[rt],regname[rs]); - assert(0); + assem_debug("adds %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); + output_w32(0x2b000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_negs(u_int rs, u_int rt) +static void emit_adds64(u_int rs1, u_int rs2, u_int rt) { - assem_debug("rsbs %s,%s,#0\n",regname[rt],regname[rs]); - assert(0); + assem_debug("adds %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); + output_w32(0xab000000 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_neg(u_int rs, u_int rt) +{ + assem_debug("neg %s,%s\n",regname[rt],regname[rs]); + output_w32(0x4b000000 | rm_rn_rd(rs, WZR, rt)); } static void emit_sub(u_int rs1, u_int rs2, u_int rt) { - assem_debug("sub %s, %s, %s\n", regname[rt], regname[rs1], regname[rs2]); + assem_debug("sub %s,%s,%s\n", regname[rt], regname[rs1], regname[rs2]); output_w32(0x4b000000 | rm_imm6_rn_rd(rs2, 0, rs1, rt)); } -static void emit_subs(u_int rs1,u_int rs2,u_int rt) +static void emit_movz(u_int imm, u_int rt) { - assem_debug("subs %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + assem_debug("movz %s,#%#x\n", regname[rt], imm); + output_w32(0x52800000 | imm16_rd(imm, rt)); +} + +static void emit_movz_lsl16(u_int imm, u_int rt) +{ + assem_debug("movz %s,#%#x,lsl #16\n", regname[rt], imm); + output_w32(0x52a00000 | imm16_rd(imm, rt)); +} + +static void emit_movn(u_int imm, u_int rt) +{ + assem_debug("movn %s,#%#x\n", regname[rt], imm); + output_w32(0x12800000 | imm16_rd(imm, rt)); +} + +static void emit_movn_lsl16(u_int imm,u_int rt) +{ + assem_debug("movn %s,#%#x,lsl #16\n", regname[rt], imm); + output_w32(0x12a00000 | imm16_rd(imm, rt)); +} + +static void emit_movk(u_int imm,u_int rt) +{ + assem_debug("movk %s,#%#x\n", regname[rt], imm); + output_w32(0x72800000 | imm16_rd(imm, rt)); +} + +static void emit_movk_lsl16(u_int imm,u_int rt) +{ + assert(imm<65536); + assem_debug("movk %s, #%#x, lsl #16\n", regname[rt], imm); + output_w32(0x72a00000 | imm16_rd(imm, rt)); } static void emit_zeroreg(u_int rt) { - assem_debug("mov %s,#0\n",regname[rt]); - assert(0); + emit_movz(0, rt); } static void emit_movimm(u_int imm, u_int rt) { - assem_debug("mov %s,#%#x\n", regname[rt], imm); - if ((imm & 0xffff0000) == 0) - output_w32(0x52800000 | imm16_rd(imm, rt)); - else if ((imm & 0xffff0000) == 0xffff0000) - assert(0); + if (imm < 65536) + emit_movz(imm, rt); + else if ((~imm) < 65536) + emit_movn(~imm, rt); + else if ((imm&0xffff) == 0) + emit_movz_lsl16(imm >> 16, rt); + else if (((~imm)&0xffff) == 0) + emit_movn_lsl16(~imm >> 16, rt); + else if (is_rotated_mask(imm)) { + u_int immr, imms; + gen_logical_imm(imm, &immr, &imms); + assem_debug("orr %s,wzr,#%#x\n", regname[rt], imm); + output_w32(0x32000000 | n_immr_imms_rn_rd(0, immr, imms, WZR, rt)); + } else { - output_w32(0x52800000 | imm16_rd(imm & 0xffff, rt)); - output_w32(0x72a00000 | imm16_rd(imm >> 16, rt)); + emit_movz(imm & 0xffff, rt); + emit_movk_lsl16(imm >> 16, rt); } } @@ -281,8 +466,20 @@ static void emit_readword(void *addr, u_int rt) assert(0); } +static void emit_readdword(void *addr, u_int rt) +{ + uintptr_t offset = (u_char *)addr - (u_char *)&dynarec_local; + if (!(offset & 7) && offset <= 32760) { + assem_debug("ldr %s,[x%d+%#lx]\n", regname64[rt], FP, offset); + output_w32(0xf9400000 | imm12_rn_rd(offset >> 3, FP, rt)); + } + else + assert(0); +} + static void emit_loadreg(u_int r, u_int hr) { + int is64 = 0; assert(r < 64); if (r == 0) emit_zeroreg(hr); @@ -293,10 +490,13 @@ static void emit_loadreg(u_int r, u_int hr) //case LOREG: addr = &lo; break; case CCREG: addr = &cycle_count; break; case CSREG: addr = &Status; break; - case INVCP: addr = &invc_ptr; break; + case INVCP: addr = &invc_ptr; is64 = 1; break; default: assert(r < 34); break; } - emit_readword(addr, hr); + if (is64) + emit_readdword(addr, hr); + else + emit_readword(addr, hr); } } @@ -311,6 +511,17 @@ static void emit_writeword(u_int rt, void *addr) assert(0); } +static void emit_writedword(u_int rt, void *addr) +{ + uintptr_t offset = (u_char *)addr - (u_char *)&dynarec_local; + if (!(offset & 7) && offset <= 32760) { + assem_debug("str %s,[x%d+%#lx]\n", regname64[rt], FP, offset); + output_w32(0xf9000000 | imm12_rn_rd(offset >> 2, FP, rt)); + } + else + assert(0); +} + static void emit_storereg(u_int r, u_int hr) { assert(r < 64); @@ -326,93 +537,113 @@ static void emit_storereg(u_int r, u_int hr) static void emit_test(u_int rs, u_int rt) { - assem_debug("tst %s,%s\n",regname[rs],regname[rt]); - assert(0); + assem_debug("tst %s,%s\n", regname[rs], regname[rt]); + output_w32(0x6a000000 | rm_rn_rd(rt, rs, WZR)); } -static void emit_testimm(u_int rs,int imm) +static void emit_testimm(u_int rs, u_int imm) { + u_int immr, imms; assem_debug("tst %s,#%#x\n", regname[rs], imm); - assert(0); + assert(is_rotated_mask(imm)); // good enough for PCSX + gen_logical_imm(imm, &immr, &imms); + output_w32(0xb9000000 | n_immr_imms_rn_rd(0, immr, imms, rs, WZR)); } static void emit_testeqimm(u_int rs,int imm) { assem_debug("tsteq %s,$%d\n",regname[rs],imm); - assert(0); + assert(0); // TODO eliminate emit_testeqimm } static void emit_not(u_int rs,u_int rt) { assem_debug("mvn %s,%s\n",regname[rt],regname[rs]); - assert(0); + output_w32(0x2a200000 | rm_rn_rd(rs, WZR, rt)); } static void emit_mvnmi(u_int rs,u_int rt) { assem_debug("mvnmi %s,%s\n",regname[rt],regname[rs]); - assert(0); + assert(0); // eliminate } static void emit_and(u_int rs1,u_int rs2,u_int rt) { assem_debug("and %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x0a000000 | rm_rn_rd(rs2, rs1, rt)); } static void emit_or(u_int rs1,u_int rs2,u_int rt) { assem_debug("orr %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x2a000000 | rm_rn_rd(rs2, rs1, rt)); } static void emit_orrshl_imm(u_int rs,u_int imm,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(imm < 32); assem_debug("orr %s,%s,%s,lsl #%d\n",regname[rt],regname[rt],regname[rs],imm); - assert(0); + output_w32(0x2a000000 | rm_imm6_rn_rd(rs, imm, rt, rt)); } static void emit_orrshr_imm(u_int rs,u_int imm,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(imm < 32); assem_debug("orr %s,%s,%s,lsr #%d\n",regname[rt],regname[rt],regname[rs],imm); - assert(0); -} - -static void emit_or_and_set_flags(u_int rs1,u_int rs2,u_int rt) -{ - assem_debug("orrs %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x2a400000 | rm_imm6_rn_rd(rs, imm, rt, rt)); } static void emit_xor(u_int rs1,u_int rs2,u_int rt) { assem_debug("eor %s,%s,%s\n",regname[rt],regname[rs1],regname[rs2]); - assert(0); + output_w32(0x4a000000 | rm_rn_rd(rs2, rs1, rt)); } -static void emit_addimm(u_int rs, uintptr_t imm, u_int rt) +static void emit_addimm_s(u_int s, u_int is64, u_int rs, uintptr_t imm, u_int rt) { + unused const char *st = s ? "s" : ""; + s = s ? 0x20000000 : 0; + is64 = is64 ? 0x80000000 : 0; if (imm < 4096) { - assem_debug("add %s,%s,%#lx\n", regname[rt], regname[rs], imm); - output_w32(0x11000000 | imm12_rn_rd(imm, rs, rt)); + assem_debug("add%s %s,%s,%#lx\n", st, regname[rt], regname[rs], imm); + output_w32(0x11000000 | is64 | s | imm12_rn_rd(imm, rs, rt)); } else if (-imm < 4096) { - assem_debug("sub %s,%s,%#lx\n", regname[rt], regname[rs], imm); - output_w32(0x51000000 | imm12_rn_rd(imm, rs, rt)); + assem_debug("sub%s %s,%s,%#lx\n", st, regname[rt], regname[rs], imm); + output_w32(0x51000000 | is64 | s | imm12_rn_rd(-imm, rs, rt)); + } + else if (imm < 16777216) { + assem_debug("add %s,%s,#%#lx\n",regname[rt],regname[rt],imm&0xfff000); + output_w32(0x11400000 | is64 | imm12_rn_rd(imm >> 12, rs, rt)); + if ((imm & 0xfff) || s) { + assem_debug("add%s %s,%s,#%#lx\n",st,regname[rt],regname[rs],imm&0xfff); + output_w32(0x11000000 | is64 | s | imm12_rn_rd(imm, rt, rt)); + } + } + else if (-imm < 16777216) { + assem_debug("sub %s,%s,#%#lx\n",regname[rt],regname[rt],-imm&0xfff000); + output_w32(0x51400000 | is64 | imm12_rn_rd(-imm >> 12, rs, rt)); + if ((imm & 0xfff) || s) { + assem_debug("sub%s %s,%s,#%#lx\n",st,regname[rt],regname[rs],-imm&0xfff); + output_w32(0x51000000 | is64 | s | imm12_rn_rd(-imm & 0xfff, rt, rt)); + } } else assert(0); } +static void emit_addimm(u_int rs, uintptr_t imm, u_int rt) +{ + emit_addimm_s(0, 0, rs, imm, rt); +} + +static void emit_addimm64(u_int rs, uintptr_t imm, u_int rt) +{ + emit_addimm_s(0, 1, rs, imm, rt); +} + static void emit_addimm_and_set_flags(int imm, u_int rt) { - assert(0); + emit_addimm_s(1, 0, rt, imm, rt); } static void emit_addimm_no_flags(u_int imm,u_int rt) @@ -420,181 +651,195 @@ static void emit_addimm_no_flags(u_int imm,u_int rt) emit_addimm(rt,imm,rt); } -static void emit_adcimm(u_int rs,int imm,u_int rt) +static void emit_logicop_imm(u_int op, u_int rs, u_int imm, u_int rt) { - assem_debug("adc %s,%s,#%#x\n",regname[rt],regname[rs],imm); - assert(0); + const char *names[] = { "and", "orr", "eor", "ands" }; + const char *name = names[op]; + u_int immr, imms; + op = op << 29; + if (is_rotated_mask(imm)) { + gen_logical_imm(imm, &immr, &imms); + assem_debug("%s %s,%s,#%#x\n", name, regname[rt], regname[rs], imm); + output_w32(op | 0x12000000 | n_immr_imms_rn_rd(0, immr, imms, rs, rt)); + } + else { + if (rs == HOST_TEMPREG || rt != HOST_TEMPREG) + host_tempreg_acquire(); + emit_movimm(imm, HOST_TEMPREG); + assem_debug("%s %s,%s,%s\n", name, regname[rt], regname[rs], regname[HOST_TEMPREG]); + output_w32(op | 0x0a000000 | rm_rn_rd(HOST_TEMPREG, rs, rt)); + if (rs == HOST_TEMPREG || rt != HOST_TEMPREG) + host_tempreg_release(); + } + (void)name; } -static void emit_rscimm(u_int rs,int imm,u_int rt) +static void emit_andimm(u_int rs, u_int imm, u_int rt) { - assem_debug("rsc %s,%s,#%#x\n",regname[rt],regname[rs],imm); - assert(0); + if (imm == 0) + emit_zeroreg(rt); + else + emit_logicop_imm(0, rs, imm, rt); } -static void emit_addimm64_32(u_int rsh,u_int rsl,int imm,u_int rth,u_int rtl) +static void emit_orimm(u_int rs, u_int imm, u_int rt) { - assert(0); + if (imm == 0) { + if (rs != rt) + emit_mov(rs, rt); + } + else + emit_logicop_imm(1, rs, imm, rt); } -static void emit_andimm(u_int rs,int imm,u_int rt) +static void emit_xorimm(u_int rs, u_int imm, u_int rt) { - assert(0); + if (imm == 0) { + if (rs != rt) + emit_mov(rs, rt); + } + else + emit_logicop_imm(2, rs, imm, rt); } -static void emit_orimm(u_int rs,int imm,u_int rt) +static void emit_sbfm(u_int rs,u_int imm,u_int rt) { - assert(0); + assem_debug("sbfm %s,%s,#0,#%d\n",regname[rt],regname[rs],imm); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, 0, imm, rs, rt)); } -static void emit_xorimm(u_int rs,int imm,u_int rt) +static void emit_ubfm(u_int rs,u_int imm,u_int rt) { - assert(0); + assem_debug("ubfm %s,%s,#0,#%d\n",regname[rt],regname[rs],imm); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, 0, imm, rs, rt)); } static void emit_shlimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("lsl %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); -} - -static void emit_lsls_imm(u_int rs,int imm,u_int rt) -{ - assert(imm>0); - assert(imm<32); - assem_debug("lsls %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, (31-imm)+1, 31-imm, rs, rt)); } static unused void emit_lslpls_imm(u_int rs,int imm,u_int rt) { - assert(imm>0); - assert(imm<32); - assem_debug("lslpls %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + assert(0); // eliminate } static void emit_shrimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("lsr %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x53000000 | n_immr_imms_rn_rd(0, imm, 31, rs, rt)); } static void emit_sarimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); assem_debug("asr %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, imm, 31, rs, rt)); } static void emit_rorimm(u_int rs,u_int imm,u_int rt) { - assert(imm>0); - assert(imm<32); - assem_debug("ror %s,%s,#%d\n",regname[rt],regname[rs],imm); - assert(0); + assem_debug("ror %s,%s,#%d",regname[rt],regname[rs],imm); + output_w32(0x13800000 | rm_imm6_rn_rd(rs, imm, rs, rt)); } static void emit_signextend16(u_int rs, u_int rt) { assem_debug("sxth %s,%s\n", regname[rt], regname[rs]); - assert(0); + output_w32(0x13000000 | n_immr_imms_rn_rd(0, 0, 15, rs, rt)); } -static void emit_shl(u_int rs,u_int shift,u_int rt) +static void emit_shl(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift < 16); - assert(0); + assem_debug("lsl %s,%s,%s",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02000 | rm_rn_rd(rshift, rs, rt)); } -static void emit_shr(u_int rs,u_int shift,u_int rt) +static void emit_shr(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("lsr %s,%s,%s\n",regname[rt],regname[rs],regname[shift]); - assert(0); -} - -static void emit_sar(u_int rs,u_int shift,u_int rt) -{ - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("asr %s,%s,%s\n",regname[rt],regname[rs],regname[shift]); - assert(0); + assem_debug("lsr %s,%s,%s\n",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02400 | rm_rn_rd(rshift, rs, rt)); } -static void emit_orrshl(u_int rs,u_int shift,u_int rt) +static void emit_sar(u_int rs,u_int rshift,u_int rt) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("orr %s,%s,%s,lsl %s\n",regname[rt],regname[rt],regname[rs],regname[shift]); - assert(0); + assem_debug("asr %s,%s,%s\n",regname[rt],regname[rs],regname[rshift]); + output_w32(0x1ac02800 | rm_rn_rd(rshift, rs, rt)); } -static void emit_orrshr(u_int rs,u_int shift,u_int rt) +static void emit_cmpimm(u_int rs, u_int imm) { - assert(rs < 31); - assert(rt < 31); - assert(shift<16); - assem_debug("orr %s,%s,%s,lsr %s\n",regname[rt],regname[rt],regname[rs],regname[shift]); - assert(0); + if (imm < 4096) { + assem_debug("cmp %s,%#x\n", regname[rs], imm); + output_w32(0x71000000 | imm12_rn_rd(imm, rs, WZR)); + } + else if (-imm < 4096) { + assem_debug("cmn %s,%#x\n", regname[rs], imm); + output_w32(0x31000000 | imm12_rn_rd(-imm, rs, WZR)); + } + else if (imm < 16777216 && !(imm & 0xfff)) { + assem_debug("cmp %s,#%#x,lsl #12\n", regname[rs], imm >> 12); + output_w32(0x71400000 | imm12_rn_rd(imm >> 12, rs, WZR)); + } + else { + host_tempreg_acquire(); + emit_movimm(imm, HOST_TEMPREG); + assem_debug("cmp %s,%s\n", regname[rs], regname[HOST_TEMPREG]); + output_w32(0x6b000000 | rm_rn_rd(HOST_TEMPREG, rs, WZR)); + host_tempreg_release(); + } } -static void emit_cmpimm(u_int rs,int imm) +static void emit_cmov_imm(u_int cond0, u_int cond1, u_int imm, u_int rt) { - assert(0); + assert(imm == 0 || imm == 1); + assert(cond0 < 0x10); + assert(cond1 < 0x10); + if (imm) { + assem_debug("csinc %s,%s,%s,%s\n",regname[rt],regname[rt],regname[WZR],condname[cond1]); + output_w32(0x1a800400 | (cond1 << 12) | rm_rn_rd(WZR, rt, rt)); + } else { + assem_debug("csel %s,%s,%s,%s\n",regname[rt],regname[WZR],regname[rt],condname[cond0]); + output_w32(0x1a800000 | (cond0 << 12) | rm_rn_rd(rt, WZR, rt)); + } } -static void emit_cmovne_imm(int imm,u_int rt) +static void emit_cmovne_imm(u_int imm,u_int rt) { - assem_debug("movne %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_NE, COND_EQ, imm, rt); } -static void emit_cmovl_imm(int imm,u_int rt) +static void emit_cmovl_imm(u_int imm,u_int rt) { - assem_debug("movlt %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_LT, COND_GE, imm, rt); } static void emit_cmovb_imm(int imm,u_int rt) { - assem_debug("movcc %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_CC, COND_CS, imm, rt); } static void emit_cmovs_imm(int imm,u_int rt) { - assem_debug("movmi %s,#%#x\n",regname[rt],imm); - assert(0); + emit_cmov_imm(COND_MI, COND_PL, imm, rt); } static void emit_cmovne_reg(u_int rs,u_int rt) { - assem_debug("movne %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,ne\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_NE << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_cmovl_reg(u_int rs,u_int rt) { - assem_debug("movlt %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,lt\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_LT << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_cmovs_reg(u_int rs,u_int rt) { - assem_debug("movmi %s,%s\n",regname[rt],regname[rs]); - assert(0); + assem_debug("csel %s,%s,%s,mi\n",regname[rt],regname[rs],regname[rt]); + output_w32(0x1a800000 | (COND_MI << 12) | rm_rn_rd(rt, rs, rt)); } static void emit_slti32(u_int rs,int imm,u_int rt) @@ -616,7 +861,7 @@ static void emit_sltiu32(u_int rs,int imm,u_int rt) static void emit_cmp(u_int rs,u_int rt) { assem_debug("cmp %s,%s\n",regname[rs],regname[rt]); - assert(0); + output_w32(0x6b000000 | rm_rn_rd(rt, rs, WZR)); } static void emit_set_gz32(u_int rs, u_int rt) @@ -630,7 +875,9 @@ static void emit_set_gz32(u_int rs, u_int rt) static void emit_set_nz32(u_int rs, u_int rt) { //assem_debug("set_nz32\n"); - assert(0); + if(rs!=rt) emit_mov(rs,rt); + emit_test(rs,rs); + emit_cmovne_imm(1,rt); } static void emit_set_if_less32(u_int rs1, u_int rs2, u_int rt) @@ -651,10 +898,10 @@ static void emit_set_if_carry32(u_int rs1, u_int rs2, u_int rt) emit_cmovb_imm(1,rt); } -static void emit_call(const void *a_) +static void emit_call(const void *a) { - intptr_t diff = (u_char *)a_ - out; - assem_debug("bl %p (%p+%lx)%s\n", a_, out, diff, func_name(a)); + intptr_t diff = (u_char *)a - out; + assem_debug("bl %p (%p+%lx)%s\n", a, out, diff, func_name(a)); assert(!(diff & 3)); if (-134217728 <= diff && diff <= 134217727) output_w32(0x94000000 | ((diff >> 2) & 0x03ffffff)); @@ -662,85 +909,85 @@ static void emit_call(const void *a_) assert(0); } -#pragma GCC diagnostic ignored "-Wunused-variable" -static void emit_jmp(const void *a_) +static void emit_jmp(const void *a) { - uintptr_t a = (uintptr_t)a_; - assem_debug("b %p (%p+%lx)%s\n", a_, out, (u_char *)a_ - out, func_name(a)); - assert(0); + assem_debug("b %p (%p+%lx)%s\n", a, out, (u_char *)a - out, func_name(a)); + u_int offset = genjmp(a); + output_w32(0x14000000 | offset); } -static void emit_jne(const void *a_) +static void emit_jne(const void *a) { - uintptr_t a = (uintptr_t)a_; - assem_debug("bne %p\n", a_); - assert(0); + assem_debug("bne %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_NE); } static void emit_jeq(const void *a) { - assem_debug("beq %p\n",a); - assert(0); + assem_debug("beq %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_EQ); } static void emit_js(const void *a) { - assem_debug("bmi %p\n",a); - assert(0); + assem_debug("bmi %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_MI); } static void emit_jns(const void *a) { - assem_debug("bpl %p\n",a); - assert(0); + assem_debug("bpl %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_PL); } static void emit_jl(const void *a) { - assem_debug("blt %p\n",a); - assert(0); + assem_debug("blt %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_LT); } static void emit_jge(const void *a) { - assem_debug("bge %p\n",a); - assert(0); + assem_debug("bge %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_GE); } static void emit_jno(const void *a) { - assem_debug("bvc %p\n",a); - assert(0); + assem_debug("bvc %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_VC); } static void emit_jc(const void *a) { - assem_debug("bcs %p\n",a); - assert(0); + assem_debug("bcs %p\n", a); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_CS); } static void emit_jcc(const void *a) { assem_debug("bcc %p\n", a); - assert(0); -} - -static void emit_callreg(u_int r) -{ - assert(r < 31); - assem_debug("blx %s\n", regname[r]); - assert(0); + u_int offset = genjmpcc(a); + output_w32(0x54000000 | (offset << 5) | COND_CC); } static void emit_jmpreg(u_int r) { - assem_debug("mov pc,%s\n",regname[r]); - assert(0); + assem_debug("br %s", regname64[r]); + output_w32(0xd61f0000 | rm_rn_rd(0, r, 0)); } static void emit_retreg(u_int r) { - assem_debug("ret %s\n", r == LR ? "" : regname[r]); + assem_debug("ret %s\n", r == LR ? "" : regname64[r]); output_w32(0xd65f0000 | rm_rn_rd(0, r, 0)); } @@ -749,34 +996,101 @@ static void emit_ret(void) emit_retreg(LR); } +static void emit_adr(void *addr, u_int rt) +{ + intptr_t offset = (u_char *)addr - out; + assert(-1048576 <= offset && offset < 1048576); + assem_debug("adr x%d,#%#lx\n", rt, offset); + output_w32(0x10000000 | ((offset&0x3) << 29) | (((offset>>2)&0x7ffff) << 5) | rt); +} + static void emit_readword_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldr %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldur %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0xb8400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); +} + +static void emit_strb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("strb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_strh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("strh %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_str_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("str %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0xb8204800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_readdword_dualindexedx8(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldr %s, [%s,%s, uxtw #3]\n",regname64[rt],regname64[rs1],regname[rs2]); + output_w32(0xf8605800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38604800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrsb_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrsb %s, [%s,%s]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x38a04800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrh %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78604800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldrsh_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldrsh %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0x78a04800 | rm_rn_rd(rs2, rs1, rt)); +} + +static void emit_ldr_dualindexed(u_int rs1, u_int rs2, u_int rt) +{ + assem_debug("ldr %s, [%s,%s, uxtw]\n",regname[rt],regname64[rs1],regname[rs2]); + output_w32(0xb8604800 | rm_rn_rd(rs2, rs1, rt)); } static void emit_movsbl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrsb %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldursb %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x38c00000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movswl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrsh %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldursh %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x78c00000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movzbl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrb %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldurb %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x38400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_movzwl_indexed(int offset, u_int rs, u_int rt) { - assem_debug("ldrh %s,%s+%d\n",regname[rt],regname[rs],offset); - assert(0); + assem_debug("ldurh %s,[%s+%#x]\n",regname[rt],regname64[rs],offset); + assert(-256 <= offset && offset < 256); + output_w32(0x78400000 | imm9_rn_rt(offset&0x1ff, rs, rt)); } static void emit_writeword_indexed(u_int rt, int offset, u_int rs) @@ -832,41 +1146,15 @@ static void emit_clz(u_int rs,u_int rt) assert(0); } -// Load 2 immediates optimizing for small code size -static void emit_mov2imm_compact(int imm1,u_int rt1,int imm2,u_int rt2) -{ - assert(0); -} - -// Conditionally select one of two immediates, optimizing for small code size -// This will only be called if HAVE_CMOV_IMM is defined -static void emit_cmov2imm_e_ne_compact(int imm1,int imm2,u_int rt) -{ - assert(0); -} - // special case for checking invalid_code -static void emit_cmpmem_indexedsr12_reg(int base,u_int r,int imm) -{ - assert(imm<128&&imm>=0); - assert(r>=0&&r<16); - assem_debug("ldrb lr,%s,%s lsr #12\n",regname[base],regname[r]); - assert(0); -} - -// Used to preload hash table entries -static unused void emit_prefetchreg(u_int r) +static void emit_cmpmem_indexedsr12_reg(u_int rbase, u_int r, u_int imm) { - assem_debug("pld %s\n",regname[r]); - assert(0); -} - -// Special case for mini_ht -static void emit_ldreq_indexed(u_int rs, u_int offset, u_int rt) -{ - assert(offset<4096); - assem_debug("ldreq %s,[%s, #%#x]\n",regname[rt],regname[rs],offset); - assert(0); + host_tempreg_acquire(); + emit_shrimm(r, 12, HOST_TEMPREG); + assem_debug("ldrb %s,[%s,%s]",regname[HOST_TEMPREG],regname64[rbase],regname64[HOST_TEMPREG]); + output_w32(0x38606800 | rm_rn_rd(HOST_TEMPREG, rbase, HOST_TEMPREG)); + emit_cmpimm(HOST_TEMPREG, imm); + host_tempreg_release(); } static void emit_orrne_imm(u_int rs,int imm,u_int rt) @@ -887,26 +1175,30 @@ static unused void emit_addpl_imm(u_int rs,int imm,u_int rt) assert(0); } +static void emit_loadlp_ofs(u_int ofs, u_int rt) +{ + output_w32(0x58000000 | imm19_rt(ofs, rt)); +} + static void emit_ldst(int is_st, int is64, u_int rt, u_int rn, u_int ofs) { u_int op = 0xb9000000; - const char *ldst = is_st ? "st" : "ld"; - char rp = is64 ? 'x' : 'w'; + unused const char *ldst = is_st ? "st" : "ld"; + unused char rp = is64 ? 'x' : 'w'; assem_debug("%sr %c%d,[x%d,#%#x]\n", ldst, rp, rt, rn, ofs); is64 = is64 ? 1 : 0; assert((ofs & ((1 << (2+is64)) - 1)) == 0); ofs = (ofs >> (2+is64)); - assert(ofs <= 0xfff); if (!is_st) op |= 0x00400000; if (is64) op |= 0x40000000; - output_w32(op | (ofs << 15) | imm12_rn_rd(ofs, rn, rt)); + output_w32(op | imm12_rn_rd(ofs, rn, rt)); } static void emit_ldstp(int is_st, int is64, u_int rt1, u_int rt2, u_int rn, int ofs) { u_int op = 0x29000000; - const char *ldst = is_st ? "st" : "ld"; - char rp = is64 ? 'x' : 'w'; + unused const char *ldst = is_st ? "st" : "ld"; + unused char rp = is64 ? 'x' : 'w'; assem_debug("%sp %c%d,%c%d,[x%d,#%#x]\n", ldst, rp, rt1, rp, rt2, rn, ofs); is64 = is64 ? 1 : 0; assert((ofs & ((1 << (2+is64)) - 1)) == 0); @@ -915,7 +1207,7 @@ static void emit_ldstp(int is_st, int is64, u_int rt1, u_int rt2, u_int rn, int ofs &= 0x7f; if (!is_st) op |= 0x00400000; if (is64) op |= 0x80000000; - output_w32(op | (ofs << 15) | rm_rn_rd(rt2, rn, rt1)); + output_w32(op | imm7_rt2_rn_rt(ofs, rt2, rn, rt1)); } static void save_load_regs_all(int is_store, u_int reglist) @@ -963,57 +1255,323 @@ static void literal_pool_jumpover(int n) { } -static void emit_extjump2(u_char *addr, int target, void *linker) +// parsed by get_pointer, find_extjump_insn +static void emit_extjump2(u_char *addr, u_int target, void *linker) { - assert(0); -} + assert(((addr[3]&0xfc)==0x14) || ((addr[3]&0xff)==0x54)); // b or b.cond -static void emit_extjump(void *addr, int target) -{ - emit_extjump2(addr, target, dyna_linker); + emit_movz(target & 0xffff, 0); + emit_movk_lsl16(target >> 16, 0); + + // addr is in the current recompiled block (max 256k) + // offset shouldn't exceed +/-1MB + emit_adr(addr, 1); + emit_jmp(linker); } -static void emit_extjump_ds(void *addr, int target) +static void check_extjump2(void *src) { - emit_extjump2(addr, target, dyna_linker_ds); + u_int *ptr = src; + assert((ptr[0] & 0xffe0001f) == 0x52800000); // movz r0, #val + (void)ptr; } // put rt_val into rt, potentially making use of rs with value rs_val -static void emit_movimm_from(u_int rs_val, u_int rs, uintptr_t rt_val, u_int rt) +static void emit_movimm_from(u_int rs_val, u_int rs, u_int rt_val, u_int rt) { - intptr_t diff = rt_val - rs_val; - if (-4096 < diff && diff < 4096) + int diff = rt_val - rs_val; + if ((-4096 <= diff && diff < 4096) + || (-16777216 <= diff && diff < 16777216 && !(diff & 0xfff))) emit_addimm(rs, diff, rt); + else if (is_rotated_mask(rs_val ^ rt_val)) + emit_xorimm(rs, rs_val ^ rt_val, rt); else - // TODO: for inline_writestub, etc - assert(0); + emit_movimm(rt_val, rt); } -// return 1 if above function can do it's job cheaply +// return 1 if the above function can do it's job cheaply static int is_similar_value(u_int v1, u_int v2) { int diff = v1 - v2; - return -4096 < diff && diff < 4096; + return (-4096 <= diff && diff < 4096) + || (-16777216 <= diff && diff < 16777216 && !(diff & 0xfff)) + || is_rotated_mask(v1 ^ v2); +} + +// trashes r2 +static void pass_args64(u_int a0, u_int a1) +{ + if(a0==1&&a1==0) { + // must swap + emit_mov64(a0,2); emit_mov64(a1,1); emit_mov64(2,0); + } + else if(a0!=0&&a1==0) { + emit_mov64(a1,1); + if (a0>=0) emit_mov64(a0,0); + } + else { + if(a0>=0&&a0!=0) emit_mov64(a0,0); + if(a1>=0&&a1!=1) emit_mov64(a1,1); + } } -//#include "pcsxmem.h" +static void loadstore_extend(enum stub_type type, u_int rs, u_int rt) +{ + switch(type) { + case LOADB_STUB: emit_sbfm(rs, 7, rt); break; + case LOADBU_STUB: + case STOREB_STUB: emit_ubfm(rs, 7, rt); break; + case LOADH_STUB: emit_sbfm(rs, 15, rt); break; + case LOADHU_STUB: + case STOREH_STUB: emit_ubfm(rs, 15, rt); break; + case LOADW_STUB: + case STOREW_STUB: if (rs != rt) emit_mov(rs, rt); break; + default: assert(0); + } +} + +#include "pcsxmem.h" //#include "pcsxmem_inline.c" static void do_readstub(int n) { assem_debug("do_readstub %x\n",start+stubs[n].a*4); - assert(0); + set_jump_target(stubs[n].addr, out); + enum stub_type type = stubs[n].type; + int i = stubs[n].a; + int rs = stubs[n].b; + const struct regstat *i_regs = (void *)stubs[n].c; + u_int reglist = stubs[n].e; + const signed char *i_regmap = i_regs->regmap; + int rt; + if(itype[i]==C1LS||itype[i]==C2LS||itype[i]==LOADLR) { + rt=get_reg(i_regmap,FTEMP); + }else{ + rt=get_reg(i_regmap,rt1[i]); + } + assert(rs>=0); + int r,temp=-1,temp2=HOST_TEMPREG,regs_saved=0; + void *restore_jump = NULL, *handler_jump = NULL; + reglist|=(1<=0&&rt1[i]!=0) + reglist&=~(1<=0&&rt1[i]!=0)) { + switch(type) { + case LOADB_STUB: emit_ldrsb_dualindexed(temp2,rs,rt); break; + case LOADBU_STUB: emit_ldrb_dualindexed(temp2,rs,rt); break; + case LOADH_STUB: emit_ldrsh_dualindexed(temp2,rs,rt); break; + case LOADHU_STUB: emit_ldrh_dualindexed(temp2,rs,rt); break; + case LOADW_STUB: emit_ldr_dualindexed(temp2,rs,rt); break; + default: assert(0); + } + } + if(regs_saved) { + restore_jump=out; + emit_jmp(0); // jump to reg restore + } + else + emit_jmp(stubs[n].retaddr); // return address + set_jump_target(handler_jump, out); + + if(!regs_saved) + save_regs(reglist); + void *handler=NULL; + if(type==LOADB_STUB||type==LOADBU_STUB) + handler=jump_handler_read8; + if(type==LOADH_STUB||type==LOADHU_STUB) + handler=jump_handler_read16; + if(type==LOADW_STUB) + handler=jump_handler_read32; + assert(handler); + pass_args64(rs,temp2); + int cc=get_reg(i_regmap,CCREG); + if(cc<0) + emit_loadreg(CCREG,2); + emit_addimm(cc<0?2:cc,CLOCK_ADJUST((int)stubs[n].d+1),2); + emit_call(handler); + // (no cycle reload after read) + if(itype[i]==C1LS||itype[i]==C2LS||(rt>=0&&rt1[i]!=0)) { + loadstore_extend(type,0,rt); + } + if(restore_jump) + set_jump_target(restore_jump, out); + restore_regs(reglist); + emit_jmp(stubs[n].retaddr); } static void inline_readstub(enum stub_type type, int i, u_int addr, signed char regmap[], int target, int adj, u_int reglist) { - assert(0); + int rs=get_reg(regmap,target); + int rt=get_reg(regmap,target); + if(rs<0) rs=get_reg(regmap,-1); + assert(rs>=0); + u_int is_dynamic=0; + uintptr_t host_addr = 0; + void *handler; + int cc=get_reg(regmap,CCREG); + //if(pcsx_direct_read(type,addr,CLOCK_ADJUST(adj+1),cc,target?rs:-1,rt)) + // return; + handler = get_direct_memhandler(mem_rtab, addr, type, &host_addr); + if (handler == NULL) { + if(rt<0||rt1[i]==0) + return; + if (addr != host_addr) { + if (host_addr >= 0x100000000ull) + abort(); // ROREG not implemented + emit_movimm_from(addr, rs, host_addr, rs); + } + switch(type) { + case LOADB_STUB: emit_movsbl_indexed(0,rs,rt); break; + case LOADBU_STUB: emit_movzbl_indexed(0,rs,rt); break; + case LOADH_STUB: emit_movswl_indexed(0,rs,rt); break; + case LOADHU_STUB: emit_movzwl_indexed(0,rs,rt); break; + case LOADW_STUB: emit_readword_indexed(0,rs,rt); break; + default: assert(0); + } + return; + } + is_dynamic=pcsxmem_is_handler_dynamic(addr); + if(is_dynamic) { + if(type==LOADB_STUB||type==LOADBU_STUB) + handler=jump_handler_read8; + if(type==LOADH_STUB||type==LOADHU_STUB) + handler=jump_handler_read16; + if(type==LOADW_STUB) + handler=jump_handler_read32; + } + + // call a memhandler + if(rt>=0&&rt1[i]!=0) + reglist&=~(1<=0&&rt1[i]!=0) + loadstore_extend(type, 0, rt); + restore_regs(reglist); } static void do_writestub(int n) { assem_debug("do_writestub %x\n",start+stubs[n].a*4); - assert(0); + set_jump_target(stubs[n].addr, out); + enum stub_type type=stubs[n].type; + int i=stubs[n].a; + int rs=stubs[n].b; + struct regstat *i_regs=(struct regstat *)stubs[n].c; + u_int reglist=stubs[n].e; + signed char *i_regmap=i_regs->regmap; + int rt,r; + if(itype[i]==C1LS||itype[i]==C2LS) { + rt=get_reg(i_regmap,r=FTEMP); + }else{ + rt=get_reg(i_regmap,r=rs2[i]); + } + assert(rs>=0); + assert(rt>=0); + int rtmp,temp=-1,temp2,regs_saved=0; + void *restore_jump = NULL, *handler_jump = NULL; + int reglist2=reglist|(1<= 0x100000000ull) + abort(); // ROREG not implemented emit_movimm_from(addr, rs, host_addr, rs); - switch(type) { + } + switch (type) { case STOREB_STUB: emit_writebyte_indexed(rt, 0, rs); break; case STOREH_STUB: emit_writehword_indexed(rt, 0, rs); break; case STOREW_STUB: emit_writeword_indexed(rt, 0, rs); break; @@ -1038,25 +1599,20 @@ static void inline_writestub(enum stub_type type, int i, u_int addr, signed char // call a memhandler save_regs(reglist); - //pass_args(rs, rt); - int cc = get_reg(regmap, CCREG); - assert(cc >= 0); - emit_addimm(cc, CLOCK_ADJUST(adj+1), 2); - //emit_movimm((uintptr_t)handler, 3); - // returns new cycle_count - - emit_readword(&last_count, HOST_TEMPREG); emit_writeword(rs, &address); // some handlers still need it - emit_add(2, HOST_TEMPREG, 2); - emit_writeword(2, &Count); - emit_mov(1, 0); + loadstore_extend(type, rt, 0); + int cc, cc_use; + cc = cc_use = get_reg(regmap, CCREG); + if (cc < 0) + emit_loadreg(CCREG, (cc_use = 2)); + emit_addimm(cc_use, CLOCK_ADJUST(adj+1), 2); + + emit_call(do_memhandler_pre); emit_call(handler); - emit_readword(&next_interupt, 0); - emit_readword(&Count, 1); - emit_writeword(0, &last_count); - emit_sub(1, 0, cc); - - emit_addimm(cc,-CLOCK_ADJUST(adj+1),cc); + emit_call(do_memhandler_post); + emit_addimm(0, -CLOCK_ADJUST(adj+1), cc_use); + if (cc < 0) + emit_storereg(CCREG, cc_use); restore_regs(reglist); } @@ -1066,24 +1622,56 @@ static void do_unalignedwritestub(int n) assert(0); } -static void do_invstub(int n) +static void set_loadlp(u_int *loadl, void *lit) { - assert(0); + uintptr_t ofs = (u_char *)lit - (u_char *)loadl; + assert((*loadl & ~0x1f) == 0x58000000); + assert((ofs & 3) == 0); + assert(ofs < 0x100000); + *loadl |= (ofs >> 2) << 5; +} + +// this output is parsed by verify_dirty, get_bounds, isclean, get_clean_addr +static void do_dirty_stub_emit_args(u_int arg0) +{ + assert(slen <= MAXBLOCK); + emit_loadlp_ofs(0, 1); // ldr x1, source + emit_loadlp_ofs(0, 2); // ldr x2, copy + emit_movz(slen*4, 3); + emit_movz(arg0 & 0xffff, 0); + emit_movk_lsl16(arg0 >> 16, 0); +} + +static void do_dirty_stub_emit_literals(u_int *loadlps) +{ + set_loadlp(&loadlps[0], out); + output_w64((uintptr_t)source); + set_loadlp(&loadlps[1], out); + output_w64((uintptr_t)copy); } -void *do_dirty_stub(int i) +static void *do_dirty_stub(int i) { assem_debug("do_dirty_stub %x\n",start+i*4); - // Careful about the code output here, verify_dirty needs to parse it. - assert(0); + u_int *loadlps = (void *)out; + do_dirty_stub_emit_args(start + i*4); + emit_call(verify_code); + void *entry = out; load_regs_entry(i); - return NULL; + if (entry == out) + entry = instr_addr[i]; + emit_jmp(instr_addr[i]); + do_dirty_stub_emit_literals(loadlps); + return entry; } static void do_dirty_stub_ds() { - // Careful about the code output here, verify_dirty needs to parse it. - assert(0); + do_dirty_stub_emit_args(start + 1); + u_int *loadlps = (void *)out; + emit_call(verify_code_ds); + emit_jmp(out + 8*2); + do_dirty_stub_emit_literals(loadlps); } /* Special assem */ @@ -1112,33 +1700,52 @@ static void multdiv_assemble_arm64(int i,struct regstat *i_regs) } #define multdiv_assemble multdiv_assemble_arm64 +static void do_jump_vaddr(u_int rs) +{ + if (rs != 0) + emit_mov(rs, 0); + emit_call(get_addr_ht); + emit_jmpreg(0); +} + static void do_preload_rhash(u_int r) { // Don't need this for ARM. On x86, this puts the value 0xf8 into the // register. On ARM the hash can be done with a single instruction (below) } static void do_preload_rhtbl(u_int ht) { - emit_addimm(FP, (u_char *)&mini_ht - (u_char *)&dynarec_local, ht); + emit_addimm64(FP, (u_char *)&mini_ht - (u_char *)&dynarec_local, ht); } static void do_rhash(u_int rs,u_int rh) { emit_andimm(rs, 0xf8, rh); } -static void do_miniht_load(int ht,u_int rh) { - assem_debug("ldr %s,[%s,%s]!\n",regname[rh],regname[ht],regname[rh]); - assert(0); +static void do_miniht_load(int ht, u_int rh) { + emit_add64(ht, rh, ht); + emit_ldst(0, 0, rh, ht, 0); } -static void do_miniht_jump(u_int rs,u_int rh,int ht) { - emit_cmp(rh,rs); - emit_ldreq_indexed(ht,4,15); - //emit_jmp(jump_vaddr_reg[rs]); - assert(0); +static void do_miniht_jump(u_int rs, u_int rh, u_int ht) { + emit_cmp(rh, rs); + void *jaddr = out; + emit_jeq(0); + do_jump_vaddr(rs); + + set_jump_target(jaddr, out); + assem_debug("ldr %s,[%s,#8]\n",regname64[ht], regname64[ht]); + output_w32(0xf9400000 | imm12_rn_rd(8 >> 3, ht, ht)); + emit_jmpreg(ht); } +// parsed by set_jump_target? static void do_miniht_insert(u_int return_address,u_int rt,int temp) { - assert(0); + emit_movz_lsl16((return_address>>16)&0xffff,rt); + emit_movk(return_address&0xffff,rt); + add_to_linker(out,return_address,1); + emit_adr(out,temp); + emit_writedword(temp,&mini_ht[(return_address&0xFF)>>3][1]); + emit_writeword(rt,&mini_ht[(return_address&0xFF)>>3][0]); } static void mark_clear_cache(void *target)