X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=deps%2Flightrec%2Flightrec.c;h=f589ce85392ad386564b8e64fcfeb38517cbd5ff;hb=2e6c828e00165e0a11ee44a144c7ec8ac24a4b60;hp=d4ab419f66d0000eedd003cb5be25bed2fb396fc;hpb=fd58fa3240e46c4d6027c28f3882a4c584af7123;p=pcsx_rearmed.git diff --git a/deps/lightrec/lightrec.c b/deps/lightrec/lightrec.c index d4ab419f..f589ce85 100644 --- a/deps/lightrec/lightrec.c +++ b/deps/lightrec/lightrec.c @@ -16,6 +16,7 @@ #include "recompiler.h" #include "regcache.h" #include "optimizer.h" +#include "tlsf/tlsf.h" #include #include @@ -26,12 +27,6 @@ #include #include #include -#if ENABLE_TINYMM -#include -#endif - -#define GENMASK(h, l) \ - (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h)))) static struct block * lightrec_precompile_block(struct lightrec_state *state, u32 pc); @@ -41,20 +36,20 @@ static void lightrec_mtc2(struct lightrec_state *state, u8 reg, u32 data); static u32 lightrec_mfc2(struct lightrec_state *state, u8 reg); static void lightrec_default_sb(struct lightrec_state *state, u32 opcode, - void *host, u32 addr, u8 data) + void *host, u32 addr, u32 data) { - *(u8 *)host = data; + *(u8 *)host = (u8)data; - if (!state->invalidate_from_dma_only) + if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY)) lightrec_invalidate(state, addr, 1); } static void lightrec_default_sh(struct lightrec_state *state, u32 opcode, - void *host, u32 addr, u16 data) + void *host, u32 addr, u32 data) { - *(u16 *)host = HTOLE16(data); + *(u16 *)host = HTOLE16((u16)data); - if (!state->invalidate_from_dma_only) + if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY)) lightrec_invalidate(state, addr, 2); } @@ -63,7 +58,7 @@ static void lightrec_default_sw(struct lightrec_state *state, u32 opcode, { *(u32 *)host = HTOLE32(data); - if (!state->invalidate_from_dma_only) + if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY)) lightrec_invalidate(state, addr, 4); } @@ -85,6 +80,27 @@ static u32 lightrec_default_lw(struct lightrec_state *state, return LE32TOH(*(u32 *)host); } +static u32 lightrec_default_lwu(struct lightrec_state *state, + u32 opcode, void *host, u32 addr) +{ + u32 val; + + memcpy(&val, host, 4); + + return LE32TOH(val); +} + +static void lightrec_default_swu(struct lightrec_state *state, u32 opcode, + void *host, u32 addr, u32 data) +{ + data = HTOLE32(data); + + memcpy(host, &data, 4); + + if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY)) + lightrec_invalidate(state, addr & ~0x3, 8); +} + static const struct lightrec_mem_map_ops lightrec_default_ops = { .sb = lightrec_default_sb, .sh = lightrec_default_sh, @@ -92,6 +108,8 @@ static const struct lightrec_mem_map_ops lightrec_default_ops = { .lb = lightrec_default_lb, .lh = lightrec_default_lh, .lw = lightrec_default_lw, + .lwu = lightrec_default_lwu, + .swu = lightrec_default_swu, }; static void __segfault_cb(struct lightrec_state *state, u32 addr, @@ -99,9 +117,9 @@ static void __segfault_cb(struct lightrec_state *state, u32 addr, { lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT); pr_err("Segmentation fault in recompiled code: invalid " - "load/store at address 0x%08x\n", addr); + "load/store at address "PC_FMT"\n", addr); if (block) - pr_err("Was executing block PC 0x%08x\n", block->pc); + pr_err("Was executing block "PC_FMT"\n", block->pc); } static void lightrec_swl(struct lightrec_state *state, @@ -109,7 +127,7 @@ static void lightrec_swl(struct lightrec_state *state, u32 opcode, void *host, u32 addr, u32 data) { unsigned int shift = addr & 0x3; - unsigned int mask = GENMASK(31, (shift + 1) * 8); + unsigned int mask = shift < 3 ? GENMASK(31, (shift + 1) * 8) : 0; u32 old_data; /* Align to 32 bits */ @@ -173,7 +191,7 @@ static u32 lightrec_lwr(struct lightrec_state *state, u32 opcode, void *host, u32 addr, u32 data) { unsigned int shift = addr & 0x3; - unsigned int mask = GENMASK(31, 32 - shift * 8); + unsigned int mask = shift ? GENMASK(31, 32 - shift * 8) : 0; u32 old_data; /* Align to 32 bits */ @@ -198,30 +216,39 @@ static void lightrec_invalidate_map(struct lightrec_state *state, const struct lightrec_mem_map *map, u32 addr, u32 len) { if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM]) { - memset(&state->code_lut[lut_offset(addr)], 0, - ((len + 3) / 4) * sizeof(void *)); + memset(lut_address(state, lut_offset(addr)), 0, + ((len + 3) / 4) * lut_elm_size(state)); } } -const struct lightrec_mem_map * -lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr) +static enum psx_map +lightrec_get_map_idx(struct lightrec_state *state, u32 kaddr) { const struct lightrec_mem_map *map; unsigned int i; - u32 addr; for (i = 0; i < state->nb_maps; i++) { - const struct lightrec_mem_map *mapi = &state->maps[i]; + map = &state->maps[i]; - if (kaddr >= mapi->pc && kaddr < mapi->pc + mapi->length) { - map = mapi; - break; - } + if (kaddr >= map->pc && kaddr < map->pc + map->length) + return (enum psx_map) i; } - if (i == state->nb_maps) + return PSX_MAP_UNKNOWN; +} + +const struct lightrec_mem_map * +lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr) +{ + const struct lightrec_mem_map *map; + enum psx_map idx; + u32 addr; + + idx = lightrec_get_map_idx(state, kaddr); + if (idx == PSX_MAP_UNKNOWN) return NULL; + map = &state->maps[idx]; addr = kaddr - map->pc; while (map->mirror_of) @@ -233,40 +260,72 @@ lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr) return map; } -u32 lightrec_rw(struct lightrec_state *state, union code op, - u32 addr, u32 data, u16 *flags, struct block *block) +u32 lightrec_rw(struct lightrec_state *state, union code op, u32 base, + u32 data, u32 *flags, struct block *block, u16 offset) { const struct lightrec_mem_map *map; const struct lightrec_mem_map_ops *ops; u32 opcode = op.opcode; + bool was_tagged = true; + u16 old_flags; + u32 addr; void *host; - addr += (s16) op.i.imm; + addr = kunseg(base + (s16) op.i.imm); - map = lightrec_get_map(state, &host, kunseg(addr)); + map = lightrec_get_map(state, &host, addr); if (!map) { __segfault_cb(state, addr, block); return 0; } - if (unlikely(map->ops)) { - if (flags) - *flags |= LIGHTREC_HW_IO; + if (flags) + was_tagged = LIGHTREC_FLAGS_GET_IO_MODE(*flags); + + if (likely(!map->ops)) { + if (flags && !LIGHTREC_FLAGS_GET_IO_MODE(*flags)) { + /* Force parallel port accesses as HW accesses, because + * the direct-I/O emitters can't differenciate it. */ + if (unlikely(map == &state->maps[PSX_MAP_PARALLEL_PORT])) + *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW); + /* If the base register is 0x0, be extra suspicious. + * Some games (e.g. Sled Storm) actually do segmentation + * faults by using uninitialized pointers, which are + * later initialized to point to hardware registers. */ + else if (op.i.rs && base == 0x0) + *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW); + else + *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_DIRECT); + } - ops = map->ops; + ops = &lightrec_default_ops; + } else if (flags && + LIGHTREC_FLAGS_GET_IO_MODE(*flags) == LIGHTREC_IO_DIRECT_HW) { + ops = &lightrec_default_ops; } else { - if (flags) - *flags |= LIGHTREC_DIRECT_IO; + if (flags && !LIGHTREC_FLAGS_GET_IO_MODE(*flags)) + *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW); - ops = &lightrec_default_ops; + ops = map->ops; + } + + if (!was_tagged) { + old_flags = block_set_flags(block, BLOCK_SHOULD_RECOMPILE); + + if (!(old_flags & BLOCK_SHOULD_RECOMPILE)) { + pr_debug("Opcode of block at "PC_FMT" has been tagged" + " - flag for recompilation\n", block->pc); + + lut_write(state, lut_offset(block->pc), NULL); + } } switch (op.i.op) { case OP_SB: - ops->sb(state, opcode, host, addr, (u8) data); + ops->sb(state, opcode, host, addr, data); return 0; case OP_SH: - ops->sh(state, opcode, host, addr, (u16) data); + ops->sh(state, opcode, host, addr, data); return 0; case OP_SWL: lightrec_swl(state, ops, opcode, host, addr, data); @@ -295,6 +354,11 @@ u32 lightrec_rw(struct lightrec_state *state, union code op, return lightrec_lwl(state, ops, opcode, host, addr, data); case OP_LWR: return lightrec_lwr(state, ops, opcode, host, addr, data); + case OP_META_LWU: + return ops->lwu(state, opcode, host, addr); + case OP_META_SWU: + ops->swu(state, opcode, host, addr, data); + return 0; case OP_LW: default: return ops->lw(state, opcode, host, addr); @@ -302,11 +366,11 @@ u32 lightrec_rw(struct lightrec_state *state, union code op, } static void lightrec_rw_helper(struct lightrec_state *state, - union code op, u16 *flags, - struct block *block) + union code op, u32 *flags, + struct block *block, u16 offset) { u32 ret = lightrec_rw(state, op, state->regs.gpr[op.i.rs], - state->regs.gpr[op.i.rt], flags, block); + state->regs.gpr[op.i.rt], flags, block, offset); switch (op.i.op) { case OP_LB: @@ -316,44 +380,41 @@ static void lightrec_rw_helper(struct lightrec_state *state, case OP_LWL: case OP_LWR: case OP_LW: - if (op.i.rt) + case OP_META_LWU: + if (OPT_HANDLE_LOAD_DELAYS && unlikely(!state->in_delay_slot_n)) { + state->temp_reg = ret; + state->in_delay_slot_n = 0xff; + } else if (op.i.rt) { state->regs.gpr[op.i.rt] = ret; - default: /* fall-through */ + } + fallthrough; + default: break; } } -static void lightrec_rw_cb(struct lightrec_state *state, union code op) +static void lightrec_rw_cb(struct lightrec_state *state, u32 arg) { - lightrec_rw_helper(state, op, NULL, NULL); + lightrec_rw_helper(state, (union code) arg, NULL, NULL, 0); } static void lightrec_rw_generic_cb(struct lightrec_state *state, u32 arg) { struct block *block; struct opcode *op; - bool was_tagged; u16 offset = (u16)arg; block = lightrec_find_block_from_lut(state->block_cache, - arg >> 16, state->next_pc); + arg >> 16, state->curr_pc); if (unlikely(!block)) { - pr_err("rw_generic: No block found in LUT for PC 0x%x offset 0x%x\n", - state->next_pc, offset); + pr_err("rw_generic: No block found in LUT for "PC_FMT" offset 0x%"PRIx16"\n", + state->curr_pc, offset); + lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT); return; } op = &block->opcode_list[offset]; - was_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO); - - lightrec_rw_helper(state, op->c, &op->flags, block); - - if (!was_tagged) { - pr_debug("Opcode of block at PC 0x%08x has been tagged - flag " - "for recompilation\n", block->pc); - - block->flags |= BLOCK_SHOULD_RECOMPILE; - } + lightrec_rw_helper(state, op->c, &op->flags, block, offset); } static u32 clamp_s32(s32 val, s32 min, s32 max) @@ -361,6 +422,16 @@ static u32 clamp_s32(s32 val, s32 min, s32 max) return val < min ? min : val > max ? max : val; } +static u16 load_u16(u32 *ptr) +{ + return ((struct u16x2 *) ptr)->l; +} + +static void store_u16(u32 *ptr, u16 value) +{ + ((struct u16x2 *) ptr)->l = value; +} + static u32 lightrec_mfc2(struct lightrec_state *state, u8 reg) { s16 gteir1, gteir2, gteir3; @@ -373,37 +444,73 @@ static u32 lightrec_mfc2(struct lightrec_state *state, u8 reg) case 9: case 10: case 11: - return (s32)(s16) state->regs.cp2d[reg]; + return (s32)(s16) load_u16(&state->regs.cp2d[reg]); case 7: case 16: case 17: case 18: case 19: - return (u16) state->regs.cp2d[reg]; + return load_u16(&state->regs.cp2d[reg]); case 28: case 29: - gteir1 = (s16) state->regs.cp2d[9]; - gteir2 = (s16) state->regs.cp2d[10]; - gteir3 = (s16) state->regs.cp2d[11]; + gteir1 = (s16) load_u16(&state->regs.cp2d[9]); + gteir2 = (s16) load_u16(&state->regs.cp2d[10]); + gteir3 = (s16) load_u16(&state->regs.cp2d[11]); return clamp_s32(gteir1 >> 7, 0, 0x1f) << 0 | clamp_s32(gteir2 >> 7, 0, 0x1f) << 5 | clamp_s32(gteir3 >> 7, 0, 0x1f) << 10; case 15: reg = 14; - default: /* fall-through */ + fallthrough; + default: return state->regs.cp2d[reg]; } } u32 lightrec_mfc(struct lightrec_state *state, union code op) { + u32 val; + if (op.i.op == OP_CP0) return state->regs.cp0[op.r.rd]; - else if (op.r.rs == OP_CP2_BASIC_MFC2) - return lightrec_mfc2(state, op.r.rd); - else - return state->regs.cp2c[op.r.rd]; + + if (op.i.op == OP_SWC2) { + val = lightrec_mfc2(state, op.i.rt); + } else if (op.r.rs == OP_CP2_BASIC_MFC2) + val = lightrec_mfc2(state, op.r.rd); + else { + val = state->regs.cp2c[op.r.rd]; + + switch (op.r.rd) { + case 4: + case 12: + case 20: + case 26: + case 27: + case 29: + case 30: + val = (u32)(s16)val; + fallthrough; + default: + break; + } + } + + if (state->ops.cop2_notify) + (*state->ops.cop2_notify)(state, op.opcode, val); + + return val; +} + +static void lightrec_mfc_cb(struct lightrec_state *state, union code op) +{ + u32 rt = lightrec_mfc(state, op); + + if (op.i.op == OP_SWC2) + state->temp_reg = rt; + else if (op.r.rt) + state->regs.gpr[op.r.rt] = rt; } static void lightrec_mtc0(struct lightrec_state *state, u8 reg, u32 data) @@ -446,7 +553,7 @@ static void lightrec_mtc0(struct lightrec_state *state, u8 reg, u32 data) status = state->regs.cp0[12]; /* Handle software interrupts */ - if (!!(status & cause & 0x300) & status) + if ((!!(status & cause & 0x300)) & status) lightrec_set_exit_flags(state, LIGHTREC_EXIT_CHECK_INTERRUPT); /* Handle hardware interrupts */ @@ -492,7 +599,8 @@ static void lightrec_mtc2(struct lightrec_state *state, u8 reg, u32 data) return; case 30: state->regs.cp2d[31] = count_leading_bits((s32) data); - default: /* fall-through */ + fallthrough; + default: state->regs.cp2d[reg] = data; break; } @@ -508,30 +616,47 @@ static void lightrec_ctc2(struct lightrec_state *state, u8 reg, u32 data) case 27: case 29: case 30: - data = (s32)(s16) data; + store_u16(&state->regs.cp2c[reg], data); break; case 31: data = (data & 0x7ffff000) | !!(data & 0x7f87e000) << 31; - default: /* fall-through */ + fallthrough; + default: + state->regs.cp2c[reg] = data; break; } - - state->regs.cp2c[reg] = data; } -void lightrec_mtc(struct lightrec_state *state, union code op, u32 data) +void lightrec_mtc(struct lightrec_state *state, union code op, u8 reg, u32 data) { - if (op.i.op == OP_CP0) - lightrec_mtc0(state, op.r.rd, data); - else if (op.r.rs == OP_CP2_BASIC_CTC2) - lightrec_ctc2(state, op.r.rd, data); - else - lightrec_mtc2(state, op.r.rd, data); + if (op.i.op == OP_CP0) { + lightrec_mtc0(state, reg, data); + } else { + if (op.i.op == OP_LWC2 || op.r.rs != OP_CP2_BASIC_CTC2) + lightrec_mtc2(state, reg, data); + else + lightrec_ctc2(state, reg, data); + + if (state->ops.cop2_notify) + (*state->ops.cop2_notify)(state, op.opcode, data); + } } -static void lightrec_mtc_cb(struct lightrec_state *state, union code op) +static void lightrec_mtc_cb(struct lightrec_state *state, u32 arg) { - lightrec_mtc(state, op, state->regs.gpr[op.r.rt]); + union code op = (union code) arg; + u32 data; + u8 reg; + + if (op.i.op == OP_LWC2) { + data = state->temp_reg; + reg = op.i.rt; + } else { + data = state->regs.gpr[op.r.rt]; + reg = op.r.rd; + } + + lightrec_mtc(state, op, reg, data); } void lightrec_rfe(struct lightrec_state *state) @@ -558,38 +683,38 @@ void lightrec_cp(struct lightrec_state *state, union code op) (*state->ops.cop2_op)(state, op.opcode); } -static void lightrec_syscall_cb(struct lightrec_state *state, union code op) -{ - lightrec_set_exit_flags(state, LIGHTREC_EXIT_SYSCALL); -} - -static void lightrec_break_cb(struct lightrec_state *state, union code op) +static void lightrec_cp_cb(struct lightrec_state *state, u32 arg) { - lightrec_set_exit_flags(state, LIGHTREC_EXIT_BREAK); + lightrec_cp(state, (union code) arg); } -struct block * lightrec_get_block(struct lightrec_state *state, u32 pc) +static struct block * lightrec_get_block(struct lightrec_state *state, u32 pc) { struct block *block = lightrec_find_block(state->block_cache, pc); + u8 old_flags; if (block && lightrec_block_is_outdated(state, block)) { - pr_debug("Block at PC 0x%08x is outdated!\n", block->pc); + pr_debug("Block at "PC_FMT" is outdated!\n", block->pc); - /* Make sure the recompiler isn't processing the block we'll - * destroy */ - if (ENABLE_THREADED_COMPILER) - lightrec_recompiler_remove(state->rec, block); + old_flags = block_set_flags(block, BLOCK_IS_DEAD); + if (!(old_flags & BLOCK_IS_DEAD)) { + /* Make sure the recompiler isn't processing the block + * we'll destroy */ + if (ENABLE_THREADED_COMPILER) + lightrec_recompiler_remove(state->rec, block); + + lightrec_unregister_block(state->block_cache, block); + remove_from_code_lut(state->block_cache, block); + lightrec_free_block(state, block); + } - lightrec_unregister_block(state->block_cache, block); - remove_from_code_lut(state->block_cache, block); - lightrec_free_block(state, block); block = NULL; } if (!block) { block = lightrec_precompile_block(state, pc); if (!block) { - pr_err("Unable to recompile block at PC 0x%x\n", pc); + pr_err("Unable to recompile block at "PC_FMT"\n", pc); lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT); return NULL; } @@ -605,9 +730,10 @@ static void * get_next_block_func(struct lightrec_state *state, u32 pc) struct block *block; bool should_recompile; void *func; + int err; - for (;;) { - func = state->code_lut[lut_offset(pc)]; + do { + func = lut_read(state, lut_offset(pc)); if (func && func != state->get_next_block) break; @@ -616,23 +742,28 @@ static void * get_next_block_func(struct lightrec_state *state, u32 pc) if (unlikely(!block)) break; - if (OPT_REPLACE_MEMSET && (block->flags & BLOCK_IS_MEMSET)) { + if (OPT_REPLACE_MEMSET && + block_has_flag(block, BLOCK_IS_MEMSET)) { func = state->memset_func; break; } - should_recompile = block->flags & BLOCK_SHOULD_RECOMPILE && - !(block->flags & BLOCK_IS_DEAD); + should_recompile = block_has_flag(block, BLOCK_SHOULD_RECOMPILE) && + !block_has_flag(block, BLOCK_NEVER_COMPILE) && + !block_has_flag(block, BLOCK_IS_DEAD); if (unlikely(should_recompile)) { - pr_debug("Block at PC 0x%08x should recompile\n", pc); + pr_debug("Block at "PC_FMT" should recompile\n", pc); - lightrec_unregister(MEM_FOR_CODE, block->code_size); - - if (ENABLE_THREADED_COMPILER) + if (ENABLE_THREADED_COMPILER) { lightrec_recompiler_add(state->rec, block); - else - lightrec_compile_block(state->cstate, block); + } else { + err = lightrec_compile_block(state->cstate, block); + if (err) { + state->exit_flags = LIGHTREC_EXIT_NOMEM; + return NULL; + } + } } if (ENABLE_THREADED_COMPILER && likely(!should_recompile)) @@ -643,40 +774,146 @@ static void * get_next_block_func(struct lightrec_state *state, u32 pc) if (likely(func)) break; - if (unlikely(block->flags & BLOCK_NEVER_COMPILE)) { + if (unlikely(block_has_flag(block, BLOCK_NEVER_COMPILE))) { pc = lightrec_emulate_block(state, block, pc); } else if (!ENABLE_THREADED_COMPILER) { /* Block wasn't compiled yet - run the interpreter */ - if (block->flags & BLOCK_FULLY_TAGGED) + if (block_has_flag(block, BLOCK_FULLY_TAGGED)) pr_debug("Block fully tagged, skipping first pass\n"); else if (ENABLE_FIRST_PASS && likely(!should_recompile)) pc = lightrec_emulate_block(state, block, pc); /* Then compile it using the profiled data */ - lightrec_compile_block(state->cstate, block); + err = lightrec_compile_block(state->cstate, block); + if (err) { + state->exit_flags = LIGHTREC_EXIT_NOMEM; + return NULL; + } + } else if (unlikely(block_has_flag(block, BLOCK_IS_DEAD))) { + /* + * If the block is dead but has never been compiled, + * then its function pointer is NULL and we cannot + * execute the block. In that case, reap all the dead + * blocks now, and in the next loop we will create a + * new block. + */ + lightrec_reaper_reap(state->reaper); } else { lightrec_recompiler_add(state->rec, block); } + } while (state->exit_flags == LIGHTREC_EXIT_NORMAL + && state->current_cycle < state->target_cycle); - if (state->exit_flags != LIGHTREC_EXIT_NORMAL || - state->current_cycle >= state->target_cycle) - break; - } - - state->next_pc = pc; + state->curr_pc = pc; return func; } -static s32 c_function_wrapper(struct lightrec_state *state, s32 cycles_delta, - void (*f)(struct lightrec_state *, u32 d), - u32 d) +static void * lightrec_alloc_code(struct lightrec_state *state, size_t size) { - state->current_cycle = state->target_cycle - cycles_delta; + void *code; + + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_lock(state); + + code = tlsf_malloc(state->tlsf, size); + + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_unlock(state); + + return code; +} + +static void lightrec_realloc_code(struct lightrec_state *state, + void *ptr, size_t size) +{ + /* NOTE: 'size' MUST be smaller than the size specified during + * the allocation. */ + + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_lock(state); + + tlsf_realloc(state->tlsf, ptr, size); + + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_unlock(state); +} + +static void lightrec_free_code(struct lightrec_state *state, void *ptr) +{ + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_lock(state); + + tlsf_free(state->tlsf, ptr); + + if (ENABLE_THREADED_COMPILER) + lightrec_code_alloc_unlock(state); +} + +static char lightning_code_data[0x80000]; + +static void * lightrec_emit_code(struct lightrec_state *state, + const struct block *block, + jit_state_t *_jit, unsigned int *size) +{ + bool has_code_buffer = ENABLE_CODE_BUFFER && state->tlsf; + jit_word_t code_size, new_code_size; + void *code; + + jit_realize(); + + if (ENABLE_DISASSEMBLER) + jit_set_data(lightning_code_data, sizeof(lightning_code_data), 0); + else + jit_set_data(NULL, 0, JIT_DISABLE_DATA | JIT_DISABLE_NOTE); + + if (has_code_buffer) { + jit_get_code(&code_size); + code = lightrec_alloc_code(state, (size_t) code_size); + + if (!code) { + if (ENABLE_THREADED_COMPILER) { + /* If we're using the threaded compiler, return + * an allocation error here. The threaded + * compiler will then empty its job queue and + * request a code flush using the reaper. */ + return NULL; + } + + /* Remove outdated blocks, and try again */ + lightrec_remove_outdated_blocks(state->block_cache, block); + + pr_debug("Re-try to alloc %zu bytes...\n", code_size); + + code = lightrec_alloc_code(state, code_size); + if (!code) { + pr_err("Could not alloc even after removing old blocks!\n"); + return NULL; + } + } + + jit_set_code(code, code_size); + } + + code = jit_emit(); + + jit_get_code(&new_code_size); + lightrec_register(MEM_FOR_CODE, new_code_size); - (*f)(state, d); + if (has_code_buffer) { + lightrec_realloc_code(state, code, (size_t) new_code_size); - return state->target_cycle - state->current_cycle; + pr_debug("Creating code block at address 0x%" PRIxPTR ", " + "code size: %" PRIuPTR " new: %" PRIuPTR "\n", + (uintptr_t) code, code_size, new_code_size); + } + + *size = (unsigned int) new_code_size; + + if (state->ops.code_inv) + state->ops.code_inv(code, new_code_size); + + return code; } static struct block * generate_wrapper(struct lightrec_state *state) @@ -684,10 +921,17 @@ static struct block * generate_wrapper(struct lightrec_state *state) struct block *block; jit_state_t *_jit; unsigned int i; - int stack_ptr; - jit_word_t code_size; - jit_node_t *to_tramp, *to_fn_epilog; jit_node_t *addr[C_WRAPPERS_COUNT - 1]; + jit_node_t *to_end[C_WRAPPERS_COUNT - 1]; + u8 tmp = JIT_R1; + +#ifdef __sh__ + /* On SH, GBR-relative loads target the r0 register. + * Use it as the temporary register to factorize the move to + * JIT_R1. */ + if (LIGHTREC_REG_STATE == _GBR) + tmp = _R0; +#endif block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block)); if (!block) @@ -704,78 +948,82 @@ static struct block * generate_wrapper(struct lightrec_state *state) jit_prolog(); jit_tramp(256); - /* Add entry points; separate them by opcodes that increment - * LIGHTREC_REG_STATE (since we cannot touch other registers). - * The difference will then tell us which C function to call. */ + /* Add entry points */ for (i = C_WRAPPERS_COUNT - 1; i > 0; i--) { - jit_addi(LIGHTREC_REG_STATE, LIGHTREC_REG_STATE, __WORDSIZE / 8); + jit_ldxi(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, c_wrappers[i])); + to_end[i - 1] = jit_b(); addr[i - 1] = jit_indirect(); } + jit_ldxi(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, c_wrappers[0])); + + for (i = 0; i < C_WRAPPERS_COUNT - 1; i++) + jit_patch(to_end[i]); + jit_movr(JIT_R1, tmp); + jit_epilog(); jit_prolog(); - stack_ptr = jit_allocai(sizeof(uintptr_t) * NUM_TEMPS); - /* Save all temporaries on stack */ - for (i = 0; i < NUM_TEMPS; i++) - jit_stxi(stack_ptr + i * sizeof(uintptr_t), JIT_FP, JIT_R(i)); + for (i = 0; i < NUM_TEMPS; i++) { + if (i + FIRST_TEMP != 1) { + jit_stxi(offsetof(struct lightrec_state, wrapper_regs[i]), + LIGHTREC_REG_STATE, JIT_R(i + FIRST_TEMP)); + } + } - /* Jump to the trampoline */ - to_tramp = jit_jmpi(); + jit_getarg(JIT_R2, jit_arg()); - /* The trampoline will jump back here */ - to_fn_epilog = jit_label(); + jit_prepare(); + jit_pushargr(LIGHTREC_REG_STATE); + jit_pushargr(JIT_R2); - /* Restore temporaries from stack */ - for (i = 0; i < NUM_TEMPS; i++) - jit_ldxi(JIT_R(i), JIT_FP, stack_ptr + i * sizeof(uintptr_t)); + jit_ldxi_ui(JIT_R2, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); - jit_ret(); - jit_epilog(); + /* state->current_cycle = state->target_cycle - delta; */ + jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, LIGHTREC_REG_CYCLE); + jit_stxi_i(offsetof(struct lightrec_state, current_cycle), + LIGHTREC_REG_STATE, LIGHTREC_REG_CYCLE); - /* Trampoline entry point. - * The sole purpose of the trampoline is to cheese Lightning not to - * save/restore the callee-saved register LIGHTREC_REG_CYCLE, since we - * do want to return to the caller with this register modified. */ - jit_prolog(); - jit_tramp(256); - jit_patch(to_tramp); - - /* Retrieve the wrapper function */ - jit_ldxi(JIT_R0, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, c_wrappers)); + /* Call the wrapper function */ + jit_finishr(JIT_R1); - /* Restore LIGHTREC_REG_STATE to its correct value */ - jit_movi(LIGHTREC_REG_STATE, (uintptr_t) state); + /* delta = state->target_cycle - state->current_cycle */; + jit_ldxi_ui(LIGHTREC_REG_CYCLE, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, current_cycle)); + jit_ldxi_ui(JIT_R1, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); + jit_subr(LIGHTREC_REG_CYCLE, JIT_R1, LIGHTREC_REG_CYCLE); - jit_prepare(); - jit_pushargr(LIGHTREC_REG_STATE); - jit_pushargr(LIGHTREC_REG_CYCLE); - jit_pushargr(JIT_R0); - jit_pushargr(JIT_R1); - jit_finishi(c_function_wrapper); - jit_retval_i(LIGHTREC_REG_CYCLE); + /* Restore temporaries from stack */ + for (i = 0; i < NUM_TEMPS; i++) { + if (i + FIRST_TEMP != 1) { + jit_ldxi(JIT_R(i + FIRST_TEMP), LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, wrapper_regs[i])); + } + } - jit_patch_at(jit_jmpi(), to_fn_epilog); + jit_ret(); jit_epilog(); block->_jit = _jit; - block->function = jit_emit(); block->opcode_list = NULL; - block->flags = 0; + block->flags = BLOCK_NO_OPCODE_LIST; block->nb_ops = 0; + block->function = lightrec_emit_code(state, block, _jit, + &block->code_size); + if (!block->function) + goto err_free_block; + state->wrappers_eps[C_WRAPPERS_COUNT - 1] = block->function; for (i = 0; i < C_WRAPPERS_COUNT - 1; i++) state->wrappers_eps[i] = jit_address(addr[i]); - jit_get_code(&code_size); - lightrec_register(MEM_FOR_CODE, code_size); - - block->code_size = code_size; - if (ENABLE_DISASSEMBLER) { pr_debug("Wrapper block:\n"); jit_disassemble(); @@ -799,30 +1047,80 @@ static u32 lightrec_memset(struct lightrec_state *state) u32 length = state->regs.gpr[5] * 4; if (!map) { - pr_err("Unable to find memory map for memset target address " - "0x%x\n", kunseg_pc); + pr_err("Unable to find memory map for memset target address "PC_FMT"\n", + kunseg_pc); return 0; } - pr_debug("Calling host memset, PC 0x%x (host address 0x%" PRIxPTR ") for %u bytes\n", + pr_debug("Calling host memset, "PC_FMT" (host address 0x%"PRIxPTR") for %u bytes\n", kunseg_pc, (uintptr_t)host, length); memset(host, 0, length); - if (!state->invalidate_from_dma_only) + if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY)) lightrec_invalidate_map(state, map, kunseg_pc, length); /* Rough estimation of the number of cycles consumed */ return 8 + 5 * (length + 3 / 4); } +static u32 lightrec_check_load_delay(struct lightrec_state *state, u32 pc, u8 reg) +{ + struct block *block; + union code first_op; + + first_op = lightrec_read_opcode(state, pc); + + if (likely(!opcode_reads_register(first_op, reg))) { + state->regs.gpr[reg] = state->temp_reg; + } else { + block = lightrec_get_block(state, pc); + if (unlikely(!block)) { + pr_err("Unable to get block at "PC_FMT"\n", pc); + lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT); + pc = 0; + } else { + pc = lightrec_handle_load_delay(state, block, pc, reg); + } + } + + return pc; +} + +static void update_cycle_counter_before_c(jit_state_t *_jit) +{ + /* update state->current_cycle */ + jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); + jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE); + jit_stxi_i(offsetof(struct lightrec_state, current_cycle), + LIGHTREC_REG_STATE, JIT_R1); +} + +static void update_cycle_counter_after_c(jit_state_t *_jit) +{ + /* Recalc the delta */ + jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, current_cycle)); + jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); + jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1); +} + +static void sync_next_pc(jit_state_t *_jit) +{ + if (lightrec_store_next_pc()) { + jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, next_pc)); + } +} + static struct block * generate_dispatcher(struct lightrec_state *state) { struct block *block; jit_state_t *_jit; - jit_node_t *to_end, *to_c, *loop, *addr, *addr2, *addr3; + jit_node_t *to_end, *loop, *addr, *addr2, *addr3, *addr4, *addr5, *jmp, *jmp2; unsigned int i; - u32 offset, ram_len; - jit_word_t code_size; + u32 offset; block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block)); if (!block) @@ -838,99 +1136,169 @@ static struct block * generate_dispatcher(struct lightrec_state *state) jit_prolog(); jit_frame(256); - jit_getarg(JIT_R0, jit_arg()); + jit_getarg(LIGHTREC_REG_STATE, jit_arg()); + jit_getarg(JIT_V0, jit_arg()); + jit_getarg(JIT_V1, jit_arg()); jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg()); /* Force all callee-saved registers to be pushed on the stack */ for (i = 0; i < NUM_REGS; i++) - jit_movr(JIT_V(i), JIT_V(i)); - - /* Pass lightrec_state structure to blocks, using the last callee-saved - * register that Lightning provides */ - jit_movi(LIGHTREC_REG_STATE, (intptr_t) state); + jit_movr(JIT_V(i + FIRST_REG), JIT_V(i + FIRST_REG)); loop = jit_label(); /* Call the block's code */ - jit_jmpr(JIT_R0); + jit_jmpr(JIT_V1); if (OPT_REPLACE_MEMSET) { /* Blocks will jump here when they need to call * lightrec_memset() */ addr3 = jit_indirect(); + jit_movr(JIT_V1, LIGHTREC_REG_CYCLE); + jit_prepare(); jit_pushargr(LIGHTREC_REG_STATE); + jit_finishi(lightrec_memset); + jit_retval(LIGHTREC_REG_CYCLE); jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE, offsetof(struct lightrec_state, regs.gpr[31])); + jit_subr(LIGHTREC_REG_CYCLE, JIT_V1, LIGHTREC_REG_CYCLE); - jit_retval(JIT_R0); - jit_subr(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, JIT_R0); + if (OPT_DETECT_IMPOSSIBLE_BRANCHES || OPT_HANDLE_LOAD_DELAYS) + jmp = jit_b(); + } + + if (OPT_DETECT_IMPOSSIBLE_BRANCHES) { + /* Blocks will jump here when they reach a branch that should + * be executed with the interpreter, passing the branch's PC + * in JIT_V0 and the address of the block in JIT_V1. */ + addr4 = jit_indirect(); + + sync_next_pc(_jit); + update_cycle_counter_before_c(_jit); + + jit_prepare(); + jit_pushargr(LIGHTREC_REG_STATE); + jit_pushargr(JIT_V1); + jit_pushargr(JIT_V0); + jit_finishi(lightrec_emulate_block); + + jit_retval(JIT_V0); + + update_cycle_counter_after_c(_jit); + + if (OPT_HANDLE_LOAD_DELAYS) + jmp2 = jit_b(); + + } + + if (OPT_HANDLE_LOAD_DELAYS) { + /* Blocks will jump here when they reach a branch with a load + * opcode in its delay slot. The delay slot has already been + * executed; the load value is in (state->temp_reg), and the + * register number is in JIT_V1. + * Jump to a C function which will evaluate the branch target's + * first opcode, to make sure that it does not read the register + * in question; and if it does, handle it accordingly. */ + addr5 = jit_indirect(); + + sync_next_pc(_jit); + update_cycle_counter_before_c(_jit); + + jit_prepare(); + jit_pushargr(LIGHTREC_REG_STATE); + jit_pushargr(JIT_V0); + jit_pushargr(JIT_V1); + jit_finishi(lightrec_check_load_delay); + + jit_retval(JIT_V0); + + update_cycle_counter_after_c(_jit); } /* The block will jump here, with the number of cycles remaining in * LIGHTREC_REG_CYCLE */ addr2 = jit_indirect(); - /* Store back the next_pc to the lightrec_state structure */ - offset = offsetof(struct lightrec_state, next_pc); + sync_next_pc(_jit); + + if (OPT_HANDLE_LOAD_DELAYS && OPT_DETECT_IMPOSSIBLE_BRANCHES) + jit_patch(jmp2); + + if (OPT_REPLACE_MEMSET + && (OPT_DETECT_IMPOSSIBLE_BRANCHES || OPT_HANDLE_LOAD_DELAYS)) { + jit_patch(jmp); + } + + /* Store back the next PC to the lightrec_state structure */ + offset = offsetof(struct lightrec_state, curr_pc); jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0); /* Jump to end if state->target_cycle < state->current_cycle */ to_end = jit_blei(LIGHTREC_REG_CYCLE, 0); /* Convert next PC to KUNSEG and avoid mirrors */ - ram_len = state->maps[PSX_MAP_KERNEL_USER_RAM].length; - jit_andi(JIT_R0, JIT_V0, 0x10000000 | (ram_len - 1)); - to_c = jit_bgei(JIT_R0, ram_len); - - /* Fast path: code is running from RAM, use the code LUT */ - if (__WORDSIZE == 64) - jit_lshi(JIT_R0, JIT_R0, 1); - jit_addr(JIT_R0, JIT_R0, LIGHTREC_REG_STATE); - jit_ldxi(JIT_R0, JIT_R0, offsetof(struct lightrec_state, code_lut)); + jit_andi(JIT_V1, JIT_V0, 0x10000000 | (RAM_SIZE - 1)); + jit_rshi_u(JIT_R1, JIT_V1, 28); + jit_andi(JIT_R2, JIT_V0, BIOS_SIZE - 1); + jit_addi(JIT_R2, JIT_R2, RAM_SIZE); + jit_movnr(JIT_V1, JIT_R2, JIT_R1); + + /* If possible, use the code LUT */ + if (!lut_is_32bit(state)) + jit_lshi(JIT_V1, JIT_V1, 1); + jit_add_state(JIT_V1, JIT_V1); + + offset = offsetof(struct lightrec_state, code_lut); + if (lut_is_32bit(state)) + jit_ldxi_ui(JIT_V1, JIT_V1, offset); + else + jit_ldxi(JIT_V1, JIT_V1, offset); /* If we get non-NULL, loop */ - jit_patch_at(jit_bnei(JIT_R0, 0), loop); + jit_patch_at(jit_bnei(JIT_V1, 0), loop); + + /* The code LUT will be set to this address when the block at the target + * PC has been preprocessed but not yet compiled by the threaded + * recompiler */ + addr = jit_indirect(); /* Slow path: call C function get_next_block_func() */ - jit_patch(to_c); if (ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES) { /* We may call the interpreter - update state->current_cycle */ - jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, target_cycle)); - jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE); - jit_stxi_i(offsetof(struct lightrec_state, current_cycle), - LIGHTREC_REG_STATE, JIT_R1); + update_cycle_counter_before_c(_jit); } - /* The code LUT will be set to this address when the block at the target - * PC has been preprocessed but not yet compiled by the threaded - * recompiler */ - addr = jit_indirect(); - - /* Get the next block */ jit_prepare(); jit_pushargr(LIGHTREC_REG_STATE); jit_pushargr(JIT_V0); + + /* Save the cycles register if needed */ + if (!(ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES)) + jit_movr(JIT_V0, LIGHTREC_REG_CYCLE); + + /* Get the next block */ jit_finishi(&get_next_block_func); - jit_retval(JIT_R0); + jit_retval(JIT_V1); if (ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES) { /* The interpreter may have updated state->current_cycle and * state->target_cycle - recalc the delta */ - jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, current_cycle)); - jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, target_cycle)); - jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1); + update_cycle_counter_after_c(_jit); + } else { + jit_movr(LIGHTREC_REG_CYCLE, JIT_V0); } + /* Reset JIT_V0 to the next PC */ + jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, curr_pc)); + /* If we get non-NULL, loop */ - jit_patch_at(jit_bnei(JIT_R0, 0), loop); + jit_patch_at(jit_bnei(JIT_V1, 0), loop); /* When exiting, the recompiled code will jump to that address */ jit_note(__FILE__, __LINE__); @@ -940,17 +1308,20 @@ static struct block * generate_dispatcher(struct lightrec_state *state) jit_epilog(); block->_jit = _jit; - block->function = jit_emit(); block->opcode_list = NULL; - block->flags = 0; + block->flags = BLOCK_NO_OPCODE_LIST; block->nb_ops = 0; - jit_get_code(&code_size); - lightrec_register(MEM_FOR_CODE, code_size); - - block->code_size = code_size; + block->function = lightrec_emit_code(state, block, _jit, + &block->code_size); + if (!block->function) + goto err_free_block; state->eob_wrapper_func = jit_address(addr2); + if (OPT_DETECT_IMPOSSIBLE_BRANCHES) + state->interpreter_func = jit_address(addr4); + if (OPT_HANDLE_LOAD_DELAYS) + state->ds_check_func = jit_address(addr5); if (OPT_REPLACE_MEMSET) state->memset_func = jit_address(addr3); state->get_next_block = jit_address(addr); @@ -978,19 +1349,22 @@ union code lightrec_read_opcode(struct lightrec_state *state, u32 pc) lightrec_get_map(state, &host, kunseg(pc)); const u32 *code = (u32 *)host; - return (union code) *code; + return (union code) LE32TOH(*code); } -unsigned int lightrec_cycles_of_opcode(union code code) +unsigned int lightrec_cycles_of_opcode(const struct lightrec_state *state, + union code code) { - return 2; + return state->cycles_per_op; } -void lightrec_free_opcode_list(struct lightrec_state *state, struct block *block) +void lightrec_free_opcode_list(struct lightrec_state *state, struct opcode *ops) { + struct opcode_list *list = container_of(ops, struct opcode_list, ops); + lightrec_free(state, MEM_FOR_IR, - sizeof(*block->opcode_list) * block->nb_ops, - block->opcode_list); + sizeof(*list) + list->nb_ops * sizeof(struct opcode), + list); } static unsigned int lightrec_get_mips_block_len(const u32 *src) @@ -1012,25 +1386,28 @@ static unsigned int lightrec_get_mips_block_len(const u32 *src) static struct opcode * lightrec_disassemble(struct lightrec_state *state, const u32 *src, unsigned int *len) { - struct opcode *list; + struct opcode_list *list; unsigned int i, length; length = lightrec_get_mips_block_len(src); - list = lightrec_malloc(state, MEM_FOR_IR, sizeof(*list) * length); + list = lightrec_malloc(state, MEM_FOR_IR, + sizeof(*list) + sizeof(struct opcode) * length); if (!list) { pr_err("Unable to allocate memory\n"); return NULL; } + list->nb_ops = (u16) length; + for (i = 0; i < length; i++) { - list[i].opcode = LE32TOH(src[i]); - list[i].flags = 0; + list->ops[i].opcode = LE32TOH(src[i]); + list->ops[i].flags = 0; } *len = length * sizeof(u32); - return list; + return list->ops; } static struct block * lightrec_precompile_block(struct lightrec_state *state, @@ -1038,11 +1415,12 @@ static struct block * lightrec_precompile_block(struct lightrec_state *state, { struct opcode *list; struct block *block; - void *host; + void *host, *addr; const struct lightrec_mem_map *map = lightrec_get_map(state, &host, kunseg(pc)); const u32 *code = (u32 *) host; unsigned int length; bool fully_tagged; + u8 block_flags = 0; if (!map) return NULL; @@ -1067,9 +1445,7 @@ static struct block * lightrec_precompile_block(struct lightrec_state *state, block->next = NULL; block->flags = 0; block->code_size = 0; -#if ENABLE_THREADED_COMPILER - block->op_list_freed = (atomic_flag)ATOMIC_FLAG_INIT; -#endif + block->precompile_date = state->current_cycle; block->nb_ops = length / sizeof(u32); lightrec_optimize(state, block); @@ -1085,21 +1461,22 @@ static struct block * lightrec_precompile_block(struct lightrec_state *state, pr_debug("Block size: %hu opcodes\n", block->nb_ops); - /* If the first opcode is an 'impossible' branch, never compile the - * block */ - if (should_emulate(block->opcode_list)) - block->flags |= BLOCK_NEVER_COMPILE; - fully_tagged = lightrec_block_is_fully_tagged(block); if (fully_tagged) - block->flags |= BLOCK_FULLY_TAGGED; + block_flags |= BLOCK_FULLY_TAGGED; - if (OPT_REPLACE_MEMSET && (block->flags & BLOCK_IS_MEMSET)) - state->code_lut[lut_offset(pc)] = state->memset_func; + if (block_flags) + block_set_flags(block, block_flags); block->hash = lightrec_calculate_block_hash(block); - pr_debug("Recompile count: %u\n", state->nb_precompile++); + if (OPT_REPLACE_MEMSET && block_has_flag(block, BLOCK_IS_MEMSET)) + addr = state->memset_func; + else + addr = state->get_next_block; + lut_write(state, lut_offset(pc), addr); + + pr_debug("Blocks created: %u\n", ++state->nb_precompile); return block; } @@ -1112,8 +1489,12 @@ static bool lightrec_block_is_fully_tagged(const struct block *block) for (i = 0; i < block->nb_ops; i++) { op = &block->opcode_list[i]; - /* Verify that all load/stores of the opcode list - * Check all loads/stores of the opcode list and mark the + /* If we have one branch that must be emulated, we cannot trash + * the opcode list. */ + if (should_emulate(op)) + return false; + + /* Check all loads/stores of the opcode list and mark the * block as fully compiled if they all have been tagged. */ switch (op->c.i.op) { case OP_LB: @@ -1130,10 +1511,12 @@ static bool lightrec_block_is_fully_tagged(const struct block *block) case OP_SWR: case OP_LWC2: case OP_SWC2: - if (!(op->flags & (LIGHTREC_DIRECT_IO | - LIGHTREC_HW_IO))) + case OP_META_LWU: + case OP_META_SWU: + if (!LIGHTREC_FLAGS_GET_IO_MODE(op->flags)) return false; - default: /* fall-through */ + fallthrough; + default: continue; } } @@ -1145,7 +1528,7 @@ static void lightrec_reap_block(struct lightrec_state *state, void *data) { struct block *block = data; - pr_debug("Reap dead block at PC 0x%08x\n", block->pc); + pr_debug("Reap dead block at "PC_FMT"\n", block->pc); lightrec_unregister_block(state->block_cache, block); lightrec_free_block(state, block); } @@ -1155,37 +1538,63 @@ static void lightrec_reap_jit(struct lightrec_state *state, void *data) _jit_destroy_state(data); } +static void lightrec_free_function(struct lightrec_state *state, void *fn) +{ + if (ENABLE_CODE_BUFFER && state->tlsf) { + pr_debug("Freeing code block at 0x%" PRIxPTR "\n", (uintptr_t) fn); + lightrec_free_code(state, fn); + } +} + +static void lightrec_reap_function(struct lightrec_state *state, void *data) +{ + lightrec_free_function(state, data); +} + +static void lightrec_reap_opcode_list(struct lightrec_state *state, void *data) +{ + lightrec_free_opcode_list(state, data); +} + int lightrec_compile_block(struct lightrec_cstate *cstate, struct block *block) { struct lightrec_state *state = cstate->state; struct lightrec_branch_target *target; - bool op_list_freed = false, fully_tagged = false; + bool fully_tagged = false; struct block *block2; struct opcode *elm; jit_state_t *_jit, *oldjit; jit_node_t *start_of_block; bool skip_next = false; - jit_word_t code_size; + void *old_fn, *new_fn; + size_t old_code_size; unsigned int i, j; + u8 old_flags; u32 offset; fully_tagged = lightrec_block_is_fully_tagged(block); if (fully_tagged) - block->flags |= BLOCK_FULLY_TAGGED; + block_set_flags(block, BLOCK_FULLY_TAGGED); _jit = jit_new_state(); if (!_jit) return -ENOMEM; oldjit = block->_jit; + old_fn = block->function; + old_code_size = block->code_size; block->_jit = _jit; lightrec_regcache_reset(cstate->reg_cache); + + if (OPT_PRELOAD_PC && (block->flags & BLOCK_PRELOAD_PC)) + lightrec_preload_pc(cstate->reg_cache, _jit); + cstate->cycles = 0; - cstate->nb_branches = 0; cstate->nb_local_branches = 0; cstate->nb_targets = 0; + cstate->no_load_delay = false; jit_prolog(); jit_tramp(256); @@ -1200,18 +1609,15 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, continue; } - cstate->cycles += lightrec_cycles_of_opcode(elm->c); - if (should_emulate(elm)) { pr_debug("Branch at offset 0x%x will be emulated\n", i << 2); - lightrec_emit_eob(cstate, block, i, false); - skip_next = !(elm->flags & LIGHTREC_NO_DS); + lightrec_emit_jump_to_interpreter(cstate, block, i); + skip_next = !op_flag_no_ds(elm->flags); } else { lightrec_rec_opcode(cstate, block, i); - skip_next = has_delay_slot(elm->c) && - !(elm->flags & LIGHTREC_NO_DS); + skip_next = !op_flag_no_ds(elm->flags) && has_delay_slot(elm->c); #if _WIN32 /* FIXME: GNU Lightning on Windows seems to use our * mapped registers as temporaries. Until the actual bug @@ -1220,10 +1626,9 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, lightrec_regcache_mark_live(cstate->reg_cache, _jit); #endif } - } - for (i = 0; i < cstate->nb_branches; i++) - jit_patch(cstate->branches[i]); + cstate->cycles += lightrec_cycles_of_opcode(state, elm->c); + } for (i = 0; i < cstate->nb_local_branches; i++) { struct lightrec_branch *branch = &cstate->local_branches[i]; @@ -1248,26 +1653,32 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, pr_err("Unable to find branch target\n"); } - jit_ldxi(JIT_R0, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, eob_wrapper_func)); - - jit_jmpr(JIT_R0); - jit_ret(); jit_epilog(); - block->function = jit_emit(); - block->flags &= ~BLOCK_SHOULD_RECOMPILE; + new_fn = lightrec_emit_code(state, block, _jit, &block->code_size); + if (!new_fn) { + if (!ENABLE_THREADED_COMPILER) + pr_err("Unable to compile block!\n"); + block->_jit = oldjit; + jit_clear_state(); + _jit_destroy_state(_jit); + return -ENOMEM; + } + + /* Pause the reaper, because lightrec_reset_lut_offset() may try to set + * the old block->function pointer to the code LUT. */ + if (ENABLE_THREADED_COMPILER) + lightrec_reaper_pause(state->reaper); + + block->function = new_fn; + block_clear_flags(block, BLOCK_SHOULD_RECOMPILE); /* Add compiled function to the LUT */ - state->code_lut[lut_offset(block->pc)] = block->function; + lut_write(state, lut_offset(block->pc), block->function); - if (ENABLE_THREADED_COMPILER) { - /* Since we might try to reap the same block multiple times, - * we need the reaper to wait until everything has been - * submitted, so that the duplicate entries can be dropped. */ - lightrec_reaper_pause(state->reaper); - } + if (ENABLE_THREADED_COMPILER) + lightrec_reaper_continue(state->reaper); /* Detect old blocks that have been covered by the new one */ for (i = 0; i < cstate->nb_targets; i++) { @@ -1277,6 +1688,13 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, continue; offset = block->pc + target->offset * sizeof(u32); + + /* Pause the reaper while we search for the block until we set + * the BLOCK_IS_DEAD flag, otherwise the block may be removed + * under our feet. */ + if (ENABLE_THREADED_COMPILER) + lightrec_reaper_pause(state->reaper); + block2 = lightrec_find_block(state->block_cache, offset); if (block2) { /* No need to check if block2 is compilable - it must @@ -1284,12 +1702,16 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, /* Set the "block dead" flag to prevent the dynarec from * recompiling this block */ - block2->flags |= BLOCK_IS_DEAD; + old_flags = block_set_flags(block2, BLOCK_IS_DEAD); + } + + if (ENABLE_THREADED_COMPILER) { + lightrec_reaper_continue(state->reaper); /* If block2 was pending for compilation, cancel it. * If it's being compiled right now, wait until it * finishes. */ - if (ENABLE_THREADED_COMPILER) + if (block2) lightrec_recompiler_remove(state->rec, block2); } @@ -1297,32 +1719,24 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, * be compiled. We can override the LUT entry with our new * block's entry point. */ offset = lut_offset(block->pc) + target->offset; - state->code_lut[offset] = jit_address(target->label); + lut_write(state, offset, jit_address(target->label)); if (block2) { pr_debug("Reap block 0x%08x as it's covered by block " "0x%08x\n", block2->pc, block->pc); /* Finally, reap the block. */ - if (ENABLE_THREADED_COMPILER) { + if (!ENABLE_THREADED_COMPILER) { + lightrec_unregister_block(state->block_cache, block2); + lightrec_free_block(state, block2); + } else if (!(old_flags & BLOCK_IS_DEAD)) { lightrec_reaper_add(state->reaper, lightrec_reap_block, block2); - } else { - lightrec_unregister_block(state->block_cache, block2); - lightrec_free_block(state, block2); } } } - if (ENABLE_THREADED_COMPILER) - lightrec_reaper_continue(state->reaper); - - jit_get_code(&code_size); - lightrec_register(MEM_FOR_CODE, code_size); - - block->code_size = code_size; - if (ENABLE_DISASSEMBLER) { pr_debug("Compiling block at PC: 0x%08x\n", block->pc); jit_disassemble(); @@ -1330,28 +1744,41 @@ int lightrec_compile_block(struct lightrec_cstate *cstate, jit_clear_state(); -#if ENABLE_THREADED_COMPILER if (fully_tagged) - op_list_freed = atomic_flag_test_and_set(&block->op_list_freed); -#endif - if (fully_tagged && !op_list_freed) { - pr_debug("Block PC 0x%08x is fully tagged" + old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST); + + if (fully_tagged && !(old_flags & BLOCK_NO_OPCODE_LIST)) { + pr_debug("Block "PC_FMT" is fully tagged" " - free opcode list\n", block->pc); - lightrec_free_opcode_list(state, block); - block->opcode_list = NULL; + + if (ENABLE_THREADED_COMPILER) { + lightrec_reaper_add(state->reaper, + lightrec_reap_opcode_list, + block->opcode_list); + } else { + lightrec_free_opcode_list(state, block->opcode_list); + } } if (oldjit) { pr_debug("Block 0x%08x recompiled, reaping old jit context.\n", block->pc); - if (ENABLE_THREADED_COMPILER) + if (ENABLE_THREADED_COMPILER) { lightrec_reaper_add(state->reaper, lightrec_reap_jit, oldjit); - else + lightrec_reaper_add(state->reaper, + lightrec_reap_function, old_fn); + } else { _jit_destroy_state(oldjit); + lightrec_free_function(state, old_fn); + } + + lightrec_unregister(MEM_FOR_CODE, old_code_size); } + pr_debug("Blocks compiled: %u\n", ++state->nb_compile); + return 0; } @@ -1371,7 +1798,7 @@ static void lightrec_print_info(struct lightrec_state *state) u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle) { - s32 (*func)(void *, s32) = (void *)state->dispatcher->function; + s32 (*func)(struct lightrec_state *, u32, void *, s32) = (void *)state->dispatcher->function; void *block_trace; s32 cycles_delta; @@ -1382,13 +1809,14 @@ u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle) target_cycle = UINT_MAX; state->target_cycle = target_cycle; - state->next_pc = pc; + state->curr_pc = pc; block_trace = get_next_block_func(state, pc); if (block_trace) { cycles_delta = state->target_cycle - state->current_cycle; - cycles_delta = (*func)(block_trace, cycles_delta); + cycles_delta = (*func)(state, state->curr_pc, + block_trace, cycles_delta); state->current_cycle = state->target_cycle - cycles_delta; } @@ -1399,23 +1827,27 @@ u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle) if (LOG_LEVEL >= INFO_L) lightrec_print_info(state); - return state->next_pc; -} - -u32 lightrec_execute_one(struct lightrec_state *state, u32 pc) -{ - return lightrec_execute(state, pc, state->current_cycle); + return state->curr_pc; } -u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc) +u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc, + u32 target_cycle) { - struct block *block = lightrec_get_block(state, pc); - if (!block) - return 0; + struct block *block; state->exit_flags = LIGHTREC_EXIT_NORMAL; + state->target_cycle = target_cycle; + + do { + block = lightrec_get_block(state, pc); + if (!block) + break; + + pc = lightrec_emulate_block(state, block, pc); - pc = lightrec_emulate_block(state, block, pc); + if (ENABLE_THREADED_COMPILER) + lightrec_reaper_reap(state->reaper); + } while (state->current_cycle < state->target_cycle); if (LOG_LEVEL >= INFO_L) lightrec_print_info(state); @@ -1425,12 +1857,19 @@ u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc) void lightrec_free_block(struct lightrec_state *state, struct block *block) { + u8 old_flags; + lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32)); - if (block->opcode_list) - lightrec_free_opcode_list(state, block); + old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST); + + if (!(old_flags & BLOCK_NO_OPCODE_LIST)) + lightrec_free_opcode_list(state, block->opcode_list); if (block->_jit) _jit_destroy_state(block->_jit); - lightrec_unregister(MEM_FOR_CODE, block->code_size); + if (block->function) { + lightrec_free_function(state, block->function); + lightrec_unregister(MEM_FOR_CODE, block->code_size); + } lightrec_free(state, MEM_FOR_IR, sizeof(*block), block); } @@ -1464,7 +1903,12 @@ struct lightrec_state * lightrec_init(char *argv0, size_t nb, const struct lightrec_ops *ops) { + const struct lightrec_mem_map *codebuf_map = &map[PSX_MAP_CODE_BUFFER]; struct lightrec_state *state; + uintptr_t addr; + void *tlsf = NULL; + bool with_32bit_lut = false; + size_t lut_size; /* Sanity-check ops */ if (!ops || !ops->cop2_op || !ops->enable_ram) { @@ -1472,25 +1916,47 @@ struct lightrec_state * lightrec_init(char *argv0, return NULL; } + if (ops->cop2_notify) + pr_debug("Optional cop2_notify callback in lightrec_ops\n"); + else + pr_debug("No optional cop2_notify callback in lightrec_ops\n"); + + if (ENABLE_CODE_BUFFER && nb > PSX_MAP_CODE_BUFFER + && codebuf_map->address) { + tlsf = tlsf_create_with_pool(codebuf_map->address, + codebuf_map->length); + if (!tlsf) { + pr_err("Unable to initialize code buffer\n"); + return NULL; + } + + if (__WORDSIZE == 64) { + addr = (uintptr_t) codebuf_map->address + codebuf_map->length - 1; + with_32bit_lut = addr == (u32) addr; + } + } + + if (with_32bit_lut) + lut_size = CODE_LUT_SIZE * 4; + else + lut_size = CODE_LUT_SIZE * sizeof(void *); + init_jit(argv0); - state = calloc(1, sizeof(*state) + - sizeof(*state->code_lut) * CODE_LUT_SIZE); + state = calloc(1, sizeof(*state) + lut_size); if (!state) goto err_finish_jit; - lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) + - sizeof(*state->code_lut) * CODE_LUT_SIZE); + lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) + lut_size); -#if ENABLE_TINYMM - state->tinymm = tinymm_init(malloc, free, 4096); - if (!state->tinymm) - goto err_free_state; -#endif + state->tlsf = tlsf; + state->with_32bit_lut = with_32bit_lut; + state->in_delay_slot_n = 0xff; + state->cycles_per_op = 2; state->block_cache = lightrec_blockcache_init(state); if (!state->block_cache) - goto err_free_tinymm; + goto err_free_state; if (ENABLE_THREADED_COMPILER) { state->rec = lightrec_recompiler_init(state); @@ -1521,10 +1987,9 @@ struct lightrec_state * lightrec_init(char *argv0, state->c_wrappers[C_WRAPPER_RW] = lightrec_rw_cb; state->c_wrappers[C_WRAPPER_RW_GENERIC] = lightrec_rw_generic_cb; + state->c_wrappers[C_WRAPPER_MFC] = lightrec_mfc_cb; state->c_wrappers[C_WRAPPER_MTC] = lightrec_mtc_cb; - state->c_wrappers[C_WRAPPER_CP] = lightrec_cp; - state->c_wrappers[C_WRAPPER_SYSCALL] = lightrec_syscall_cb; - state->c_wrappers[C_WRAPPER_BREAK] = lightrec_break_cb; + state->c_wrappers[C_WRAPPER_CP] = lightrec_cp_cb; map = &state->maps[PSX_MAP_BIOS]; state->offset_bios = (uintptr_t)map->address - map->pc; @@ -1532,6 +1997,9 @@ struct lightrec_state * lightrec_init(char *argv0, map = &state->maps[PSX_MAP_SCRATCH_PAD]; state->offset_scratch = (uintptr_t)map->address - map->pc; + map = &state->maps[PSX_MAP_HW_REGISTERS]; + state->offset_io = (uintptr_t)map->address - map->pc; + map = &state->maps[PSX_MAP_KERNEL_USER_RAM]; state->offset_ram = (uintptr_t)map->address - map->pc; @@ -1543,12 +2011,16 @@ struct lightrec_state * lightrec_init(char *argv0, if (state->offset_bios == 0 && state->offset_scratch == 0 && state->offset_ram == 0 && + state->offset_io == 0 && state->mirrors_mapped) { pr_info("Memory map is perfect. Emitted code will be best.\n"); } else { pr_info("Memory map is sub-par. Emitted code will be slow.\n"); } + if (state->with_32bit_lut) + pr_info("Using 32-bit LUT\n"); + return state; err_free_dispatcher: @@ -1563,16 +2035,14 @@ err_free_recompiler: lightrec_free_cstate(state->cstate); err_free_block_cache: lightrec_free_block_cache(state->block_cache); -err_free_tinymm: -#if ENABLE_TINYMM - tinymm_shutdown(state->tinymm); err_free_state: -#endif lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) + - sizeof(*state->code_lut) * CODE_LUT_SIZE); + lut_elm_size(state) * CODE_LUT_SIZE); free(state); err_finish_jit: finish_jit(); + if (ENABLE_CODE_BUFFER && tlsf) + tlsf_destroy(tlsf); return NULL; } @@ -1582,6 +2052,10 @@ void lightrec_destroy(struct lightrec_state *state) state->current_cycle = ~state->current_cycle; lightrec_print_info(state); + lightrec_free_block_cache(state->block_cache); + lightrec_free_block(state, state->dispatcher); + lightrec_free_block(state, state->c_wrapper_block); + if (ENABLE_THREADED_COMPILER) { lightrec_free_recompiler(state->rec); lightrec_reaper_destroy(state->reaper); @@ -1589,46 +2063,48 @@ void lightrec_destroy(struct lightrec_state *state) lightrec_free_cstate(state->cstate); } - lightrec_free_block_cache(state->block_cache); - lightrec_free_block(state, state->dispatcher); - lightrec_free_block(state, state->c_wrapper_block); finish_jit(); + if (ENABLE_CODE_BUFFER && state->tlsf) + tlsf_destroy(state->tlsf); -#if ENABLE_TINYMM - tinymm_shutdown(state->tinymm); -#endif lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) + - sizeof(*state->code_lut) * CODE_LUT_SIZE); + lut_elm_size(state) * CODE_LUT_SIZE); free(state); } void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len) { u32 kaddr = kunseg(addr & ~0x3); - const struct lightrec_mem_map *map = lightrec_get_map(state, NULL, kaddr); - - if (map) { - if (map != &state->maps[PSX_MAP_KERNEL_USER_RAM]) - return; + enum psx_map idx = lightrec_get_map_idx(state, kaddr); + switch (idx) { + case PSX_MAP_MIRROR1: + case PSX_MAP_MIRROR2: + case PSX_MAP_MIRROR3: /* Handle mirrors */ - kaddr &= (state->maps[PSX_MAP_KERNEL_USER_RAM].length - 1); - - lightrec_invalidate_map(state, map, kaddr, len); + kaddr &= RAM_SIZE - 1; + fallthrough; + case PSX_MAP_KERNEL_USER_RAM: + break; + default: + return; } + + memset(lut_address(state, lut_offset(kaddr)), 0, + ((len + 3) / 4) * lut_elm_size(state)); } void lightrec_invalidate_all(struct lightrec_state *state) { - memset(state->code_lut, 0, sizeof(*state->code_lut) * CODE_LUT_SIZE); + memset(state->code_lut, 0, lut_elm_size(state) * CODE_LUT_SIZE); } -void lightrec_set_invalidate_mode(struct lightrec_state *state, bool dma_only) +void lightrec_set_unsafe_opt_flags(struct lightrec_state *state, u32 flags) { - if (state->invalidate_from_dma_only != dma_only) + if ((flags ^ state->opt_flags) & LIGHTREC_OPT_INV_DMA_ONLY) lightrec_invalidate_all(state); - state->invalidate_from_dma_only = dma_only; + state->opt_flags = flags; } void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags) @@ -1671,3 +2147,8 @@ struct lightrec_registers * lightrec_get_registers(struct lightrec_state *state) { return &state->regs; } + +void lightrec_set_cycles_per_opcode(struct lightrec_state *state, u32 cycles) +{ + state->cycles_per_op = cycles; +}