X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=deps%2Flightrec%2Femitter.c;h=02510f0e075d7009a8dc1fd9d9053181a4f1709c;hb=0e720fb13f7933930608b62854ea8f4a9d97c934;hp=3f7da5856641106c96345a77cad2ed70a4762867;hpb=13b02197fcb7575646408094d5583ed7391b1153;p=pcsx_rearmed.git diff --git a/deps/lightrec/emitter.c b/deps/lightrec/emitter.c index 3f7da585..02510f0e 100644 --- a/deps/lightrec/emitter.c +++ b/deps/lightrec/emitter.c @@ -21,6 +21,12 @@ static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset); static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset); static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset); +static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset); +static void rec_cp2_do_mtc2(struct lightrec_cstate *state, + const struct block *block, u16 offset, u8 reg, u8 in_reg); +static void rec_cp2_do_mfc2(struct lightrec_cstate *state, + const struct block *block, u16 offset, + u8 reg, u8 out_reg); static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset) { @@ -30,12 +36,34 @@ static void unknown_opcode(struct lightrec_cstate *state, const struct block *bl } static void -lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit) +lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void)) { /* Prevent jit_jmpi() from using our cycles register as a temporary */ jit_live(LIGHTREC_REG_CYCLE); - jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func); + jit_patch_abs(jit_jmpi(), fn); +} + +static void +lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit) +{ + lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func); +} + +static void +lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit) +{ + lightrec_jump_to_fn(_jit, state->state->ds_check_func); +} + +static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit, + u8 ra_reg, u32 pc, u32 link) +{ + u8 link_reg; + + link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0); + lightrec_load_imm(reg_cache, _jit, link_reg, pc, link); + lightrec_free_reg(reg_cache, link_reg); } static void lightrec_emit_end_of_block(struct lightrec_cstate *state, @@ -46,89 +74,117 @@ static void lightrec_emit_end_of_block(struct lightrec_cstate *state, struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; const struct opcode *op = &block->opcode_list[offset], - *next = &block->opcode_list[offset + 1]; + *ds = get_delay_slot(block->opcode_list, offset); u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c); jit_note(__FILE__, __LINE__); - if (link) { - /* Update the $ra register */ - u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0); - jit_movi(link_reg, link); - lightrec_free_reg(reg_cache, link_reg); - } + if (link && ra_reg != reg_new_pc) + update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); - if (reg_new_pc < 0) { - reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0); - lightrec_lock_reg(reg_cache, _jit, reg_new_pc); + if (reg_new_pc < 0) + lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm); + else + lightrec_load_next_pc(reg_cache, _jit, reg_new_pc); - jit_movi(reg_new_pc, imm); + if (link && ra_reg == reg_new_pc) { + /* Handle the special case: JALR $r0, $r0 + * In that case the target PC should be the old value of the + * register. */ + update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); } if (has_delay_slot(op->c) && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) { - cycles += lightrec_cycles_of_opcode(next->c); + cycles += lightrec_cycles_of_opcode(ds->c); /* Recompile the delay slot */ - if (next->c.opcode) + if (ds->c.opcode) lightrec_rec_opcode(state, block, offset + 1); } /* Clean the remaining registers */ lightrec_clean_regs(reg_cache, _jit); - jit_movr(JIT_V0, reg_new_pc); - if (cycles && update_cycles) { jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); pr_debug("EOB: %u cycles\n", cycles); } - lightrec_jump_to_eob(state, _jit); + if (op_flag_load_delay(ds->flags) + && opcode_is_load(ds->c) && !state->no_load_delay) { + /* If the delay slot is a load opcode, its target register + * will be written after the first opcode of the target is + * executed. Handle this by jumping to a special section of + * the dispatcher. It expects the loaded value to be in + * REG_TEMP, and the target register number to be in JIT_V1.*/ + jit_movi(JIT_V1, ds->c.i.rt); + + lightrec_jump_to_ds_check(state, _jit); + } else { + lightrec_jump_to_eob(state, _jit); + } + + lightrec_regcache_reset(reg_cache); } -void lightrec_emit_eob(struct lightrec_cstate *state, - const struct block *block, u16 offset) +void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state, + const struct block *block, u16 offset) { struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; lightrec_clean_regs(reg_cache, _jit); - jit_movi(JIT_V0, block->pc + (offset << 2)); - jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); + /* Call the interpreter with the block's address in JIT_V1 and the + * PC (which might have an offset) in JIT_V0. */ + lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, + block->pc + (offset << 2)); + if (lightrec_store_next_pc()) { + jit_stxi_i(offsetof(struct lightrec_state, next_pc), + LIGHTREC_REG_STATE, JIT_V0); + } - lightrec_jump_to_eob(state, _jit); + jit_movi(JIT_V1, (uintptr_t)block); + + jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); + lightrec_jump_to_fn(_jit, state->state->interpreter_func); } -static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset) +static void lightrec_emit_eob(struct lightrec_cstate *state, + const struct block *block, u16 offset) { struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; - const struct opcode *op = &block->opcode_list[offset]; - u8 rs; - rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0); - lightrec_lock_reg(reg_cache, _jit, rs); + lightrec_clean_regs(reg_cache, _jit); + + lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, + block->pc + (offset << 2)); + if (lightrec_store_next_pc()) { + jit_stxi_i(offsetof(struct lightrec_state, next_pc), + LIGHTREC_REG_STATE, JIT_V0); + } + + jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); - return rs; + lightrec_jump_to_eob(state, _jit); } static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset) { - u8 rs = get_jr_jalr_reg(state, block, offset); + union code c = block->opcode_list[offset].c; _jit_name(block->_jit, __func__); - lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true); + lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true); } static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset) { - u8 rs = get_jr_jalr_reg(state, block, offset); union code c = block->opcode_list[offset].c; _jit_name(block->_jit, __func__); - lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd, + lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd, get_branch_pc(block, offset, 2), true); } @@ -197,19 +253,19 @@ static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 jit_state_t *_jit = block->_jit; struct lightrec_branch *branch; const struct opcode *op = &block->opcode_list[offset], - *next = &block->opcode_list[offset + 1]; + *ds = get_delay_slot(block->opcode_list, offset); jit_node_t *addr; - u8 link_reg, rs, rt; - bool is_forward = (s16)op->i.imm >= -1; + bool is_forward = (s16)op->i.imm >= 0; int op_cycles = lightrec_cycles_of_opcode(op->c); u32 target_offset, cycles = state->cycles + op_cycles; bool no_indirection = false; u32 next_pc; + u8 rs, rt; jit_note(__FILE__, __LINE__); if (!op_flag_no_ds(op->flags)) - cycles += lightrec_cycles_of_opcode(next->c); + cycles += lightrec_cycles_of_opcode(ds->c); state->cycles = -op_cycles; @@ -223,7 +279,7 @@ static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 lightrec_do_early_unload(state, block, offset); if (op_flag_local_branch(op->flags) && - (op_flag_no_ds(op->flags) || !next->opcode) && + (op_flag_no_ds(op->flags) || !ds->opcode) && is_forward && !lightrec_has_dirty_regs(reg_cache)) no_indirection = true; @@ -245,16 +301,15 @@ static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 if (op_flag_local_branch(op->flags)) { /* Recompile the delay slot */ - if (!op_flag_no_ds(op->flags) && next->opcode) + if (!op_flag_no_ds(op->flags) && ds->opcode) { + /* Never handle load delays with local branches. */ + state->no_load_delay = true; lightrec_rec_opcode(state, block, offset + 1); - - if (link) { - /* Update the $ra register */ - link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0); - jit_movi(link_reg, link); - lightrec_free_reg(reg_cache, link_reg); } + if (link) + update_ra_register(reg_cache, _jit, 31, block->pc, link); + /* Clean remaining registers */ lightrec_clean_regs(reg_cache, _jit); @@ -277,6 +332,7 @@ static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 if (!op_flag_local_branch(op->flags) || !is_forward) { next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm); + state->no_load_delay = op_flag_local_branch(op->flags); lightrec_emit_end_of_block(state, block, offset, -1, next_pc, 31, link, false); } @@ -287,16 +343,13 @@ static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 lightrec_regcache_leave_branch(reg_cache, regs_backup); - if (bz && link) { - /* Update the $ra register */ - link_reg = lightrec_alloc_reg_out(reg_cache, _jit, - 31, REG_EXT); - jit_movi(link_reg, (s32)link); - lightrec_free_reg(reg_cache, link_reg); - } + if (bz && link) + update_ra_register(reg_cache, _jit, 31, block->pc, link); - if (!op_flag_no_ds(op->flags) && next->opcode) + if (!op_flag_no_ds(op->flags) && ds->opcode) { + state->no_load_delay = true; lightrec_rec_opcode(state, block, offset + 1); + } } } @@ -942,11 +995,11 @@ static void rec_alu_div(struct lightrec_cstate *state, if (!op_flag_no_lo(flags)) { if (is_signed) { - jit_lti(lo, rs, 0); + jit_ltr(lo, rs, rt); jit_lshi(lo, lo, 1); jit_subi(lo, lo, 1); } else { - jit_movi(lo, 0xffffffff); + jit_subi(lo, rt, 1); } } @@ -1098,6 +1151,7 @@ static void rec_io(struct lightrec_cstate *state, u32 flags = block->opcode_list[offset].flags; bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags); u32 lut_entry; + u8 zero; jit_note(__FILE__, __LINE__); @@ -1108,6 +1162,16 @@ static void rec_io(struct lightrec_cstate *state, else if (load_rt) lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); + if (op_flag_load_delay(flags) && !state->no_load_delay) { + /* Clear state->in_delay_slot_n. This notifies the lightrec_rw + * wrapper that it should write the REG_TEMP register instead of + * the actual output register of the opcode. */ + zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); + jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n), + LIGHTREC_REG_STATE, zero); + lightrec_free_reg(reg_cache, zero); + } + if (is_tagged) { call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW); } else { @@ -1126,7 +1190,7 @@ static u32 rec_io_mask(const struct lightrec_state *state) { u32 length = state->maps[PSX_MAP_HW_REGISTERS].length; - return GENMASK(31 - clz32(length - 1), 0); + return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0); } static void rec_store_memory(struct lightrec_cstate *cstate, @@ -1150,8 +1214,10 @@ static void rec_store_memory(struct lightrec_cstate *cstate, ((!state->mirrors_mapped && !no_mask) || (invalidate && ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt)))); bool need_tmp = !no_mask || addr_offset || add_imm || invalidate; + bool swc2 = c.i.op == OP_SWC2; + u8 in_reg = swc2 ? REG_TEMP : c.i.rt; - rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); if (need_tmp) tmp = lightrec_alloc_reg_temp(reg_cache, _jit); @@ -1160,6 +1226,7 @@ static void rec_store_memory(struct lightrec_cstate *cstate, if (add_imm) { jit_addi(tmp, addr_reg, (s16)c.i.imm); + lightrec_free_reg(reg_cache, rs); addr_reg = tmp; imm = 0; } else if (simm) { @@ -1179,7 +1246,7 @@ static void rec_store_memory(struct lightrec_cstate *cstate, addr_reg2 = addr_reg; } - if (is_big_endian() && swap_code && c.i.rt) { + if (is_big_endian() && swap_code && in_reg) { tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); jit_new_node_ww(swap_code, tmp3, rt); @@ -1208,7 +1275,7 @@ static void rec_store_memory(struct lightrec_cstate *cstate, if (addr_reg == rs && c.i.rs == 0) { addr_reg = LIGHTREC_REG_STATE; } else { - jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE); + jit_add_state(tmp, addr_reg); addr_reg = tmp; } @@ -1273,14 +1340,16 @@ static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate, union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_end; - u8 tmp, tmp2, rs, rt; + bool swc2 = c.i.op == OP_SWC2; + bool offset_ram_or_scratch = state->offset_ram || state->offset_scratch; + u8 tmp, tmp2, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt; s16 imm; jit_note(__FILE__, __LINE__); rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - if (state->offset_ram || state->offset_scratch) + if (offset_ram_or_scratch) tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); /* Convert to KUNSEG and avoid RAM mirrors */ @@ -1312,14 +1381,14 @@ static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate, jit_movi(tmp2, state->offset_ram); } - if (state->offset_ram || state->offset_scratch) { + if (offset_ram_or_scratch) { jit_addr(tmp, tmp, tmp2); lightrec_free_reg(reg_cache, tmp2); } - rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); - if (is_big_endian() && swap_code && c.i.rt) { + if (is_big_endian() && swap_code && in_reg) { tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); jit_new_node_ww(swap_code, tmp2, rt); @@ -1343,7 +1412,9 @@ static void rec_store_direct(struct lightrec_cstate *cstate, const struct block union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_end; - u8 tmp, tmp2, tmp3, rs, rt; + bool swc2 = c.i.op == OP_SWC2; + u8 tmp, tmp2, tmp3, masked_reg, rs, rt; + u8 in_reg = swc2 ? REG_TEMP : c.i.rt; jit_note(__FILE__, __LINE__); @@ -1362,13 +1433,24 @@ static void rec_store_direct(struct lightrec_cstate *cstate, const struct block lightrec_free_reg(reg_cache, rs); tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - to_not_ram = jit_bgti(tmp2, ram_size); + if (state->offset_ram != state->offset_scratch) { + to_not_ram = jit_bgti(tmp2, ram_size); + masked_reg = tmp2; + } else { + jit_lti_u(tmp, tmp2, ram_size); + jit_movnr(tmp, tmp2, tmp); + masked_reg = tmp; + } /* Compute the offset to the code LUT */ - jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3); + if (c.i.op == OP_SW) + jit_andi(tmp, masked_reg, RAM_SIZE - 1); + else + jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3); + if (!lut_is_32bit(state)) jit_lshi(tmp, tmp, 1); - jit_addr(tmp, LIGHTREC_REG_STATE, tmp); + jit_add_state(tmp, tmp); /* Write NULL to the code LUT to invalidate any block that's there */ if (lut_is_32bit(state)) @@ -1380,10 +1462,9 @@ static void rec_store_direct(struct lightrec_cstate *cstate, const struct block jit_movi(tmp, state->offset_ram); to_end = jit_b(); + jit_patch(to_not_ram); } - jit_patch(to_not_ram); - if (state->offset_ram || state->offset_scratch) jit_movi(tmp, state->offset_scratch); @@ -1396,9 +1477,9 @@ static void rec_store_direct(struct lightrec_cstate *cstate, const struct block lightrec_free_reg(reg_cache, tmp); lightrec_free_reg(reg_cache, tmp3); - rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); - if (is_big_endian() && swap_code && c.i.rt) { + if (is_big_endian() && swap_code && in_reg) { tmp = lightrec_alloc_reg_temp(reg_cache, _jit); jit_new_node_ww(swap_code, tmp, rt); @@ -1418,10 +1499,26 @@ static void rec_store(struct lightrec_cstate *state, jit_code_t code, jit_code_t swap_code) { u32 flags = block->opcode_list[offset].flags; + u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags); bool no_invalidate = op_flag_no_invalidate(flags) || state->state->invalidate_from_dma_only; + union code c = block->opcode_list[offset].c; + bool is_swc2 = c.i.op == OP_SWC2; + + if (is_swc2) { + switch (mode) { + case LIGHTREC_IO_RAM: + case LIGHTREC_IO_SCRATCH: + case LIGHTREC_IO_DIRECT: + case LIGHTREC_IO_DIRECT_HW: + rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP); + break; + default: + break; + } + } - switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { + switch (mode) { case LIGHTREC_IO_RAM: rec_store_ram(state, block, offset, code, swap_code, !no_invalidate); @@ -1442,8 +1539,11 @@ static void rec_store(struct lightrec_cstate *state, break; default: rec_io(state, block, offset, true, false); - break; + return; } + + if (is_swc2) + lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); } static void rec_SB(struct lightrec_cstate *state, @@ -1465,7 +1565,9 @@ static void rec_SW(struct lightrec_cstate *state, const struct block *block, u16 offset) { - _jit_name(block->_jit, __func__); + union code c = block->opcode_list[offset].c; + + _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW"); rec_store(state, block, offset, jit_code_stxi_i, jit_code_bswapr_ui); } @@ -1484,13 +1586,6 @@ static void rec_SWR(struct lightrec_cstate *state, rec_io(state, block, offset, true, false); } -static void rec_SWC2(struct lightrec_cstate *state, - const struct block *block, u16 offset) -{ - _jit_name(block->_jit, __func__); - rec_io(state, block, offset, false, false); -} - static void rec_load_memory(struct lightrec_cstate *cstate, const struct block *block, u16 offset, jit_code_t code, jit_code_t swap_code, bool is_unsigned, @@ -1498,20 +1593,25 @@ static void rec_load_memory(struct lightrec_cstate *cstate, { struct regcache *reg_cache = cstate->reg_cache; struct opcode *op = &block->opcode_list[offset]; + bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; jit_state_t *_jit = block->_jit; - u8 rs, rt, addr_reg, flags = REG_EXT; + u8 rs, rt, out_reg, addr_reg, flags = REG_EXT; bool no_mask = op_flag_no_mask(op->flags); union code c = op->c; s16 imm; - if (!c.i.rt) + if (load_delay || c.i.op == OP_LWC2) + out_reg = REG_TEMP; + else if (c.i.rt) + out_reg = c.i.rt; + else return; if (is_unsigned) flags |= REG_ZEXT; rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); - rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); + rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) { jit_addi(rt, rs, (s16)c.i.imm); @@ -1594,13 +1694,19 @@ static void rec_load_direct(struct lightrec_cstate *cstate, { struct lightrec_state *state = cstate->state; struct regcache *reg_cache = cstate->reg_cache; - union code c = block->opcode_list[offset].c; + struct opcode *op = &block->opcode_list[offset]; + bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2; - u8 tmp, rs, rt, addr_reg, flags = REG_EXT; + u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT; + union code c = op->c; s16 imm; - if (!c.i.rt) + if (load_delay || c.i.op == OP_LWC2) + out_reg = REG_TEMP; + else if (c.i.rt) + out_reg = c.i.rt; + else return; if (is_unsigned) @@ -1608,7 +1714,7 @@ static void rec_load_direct(struct lightrec_cstate *cstate, jit_note(__FILE__, __LINE__); rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); - rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); + rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); if ((state->offset_ram == state->offset_bios && state->offset_ram == state->offset_scratch && @@ -1700,7 +1806,8 @@ static void rec_load(struct lightrec_cstate *state, const struct block *block, u16 offset, jit_code_t code, jit_code_t swap_code, bool is_unsigned) { - u32 flags = block->opcode_list[offset].flags; + const struct opcode *op = &block->opcode_list[offset]; + u32 flags = op->flags; switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { case LIGHTREC_IO_RAM: @@ -1720,7 +1827,12 @@ static void rec_load(struct lightrec_cstate *state, const struct block *block, break; default: rec_io(state, block, offset, false, true); - break; + return; + } + + if (op->i.op == OP_LWC2) { + rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP); + lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); } } @@ -1764,6 +1876,7 @@ static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u1 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; jit_code_t code; if (is_big_endian() && __WORDSIZE == 64) @@ -1771,16 +1884,10 @@ static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 else code = jit_code_ldxi_i; - _jit_name(block->_jit, __func__); + _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW"); rec_load(state, block, offset, code, jit_code_bswapr_ui, false); } -static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset) -{ - _jit_name(block->_jit, __func__); - rec_io(state, block, offset, false, false); -} - static void rec_break_syscall(struct lightrec_cstate *state, const struct block *block, u16 offset, u32 exit_code) @@ -1797,6 +1904,15 @@ static void rec_break_syscall(struct lightrec_cstate *state, jit_stxi_i(offsetof(struct lightrec_state, exit_flags), LIGHTREC_REG_STATE, tmp); + jit_ldxi_i(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); + jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE); + jit_movi(LIGHTREC_REG_CYCLE, 0); + jit_stxi_i(offsetof(struct lightrec_state, target_cycle), + LIGHTREC_REG_STATE, tmp); + jit_stxi_i(offsetof(struct lightrec_state, current_cycle), + LIGHTREC_REG_STATE, tmp); + lightrec_free_reg(reg_cache, tmp); /* TODO: the return address should be "pc - 4" if we're a delay slot */ @@ -1826,7 +1942,9 @@ static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u1 jit_state_t *_jit = block->_jit; jit_note(__FILE__, __LINE__); - lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); + + if (c.i.op != OP_SWC2) + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC); } @@ -1840,6 +1958,7 @@ static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u1 jit_note(__FILE__, __LINE__); lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); + lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false); call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC); @@ -1869,13 +1988,16 @@ rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset) lightrec_free_reg(reg_cache, rt); } -static bool block_in_bios(const struct lightrec_cstate *state, - const struct block *block) +static bool block_uses_icache(const struct lightrec_cstate *state, + const struct block *block) { - const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS]; + const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM]; u32 pc = kunseg(block->pc); - return pc >= bios->pc && pc < bios->pc + bios->length; + if (pc < map->pc || pc >= map->pc + map->length) + return false; + + return (block->pc >> 28) < 0xa; } static void @@ -1885,6 +2007,7 @@ rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; u8 rt, tmp = 0, tmp2, status; + jit_node_t *to_end; jit_note(__FILE__, __LINE__); @@ -1900,10 +2023,11 @@ rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) break; } - if (block_in_bios(state, block) && c.r.rd == 12) { - /* If we are running code from the BIOS, handle writes to the - * Status register in C. BIOS code may toggle bit 16 which will - * map/unmap the RAM, while game code cannot do that. */ + if (!block_uses_icache(state, block) && c.r.rd == 12) { + /* If we are not running code from the RAM through kuseg or + * kseg0, handle writes to the Status register in C; as the + * code may toggle bit 16 which isolates the cache. Code + * running from kuseg or kseg0 in RAM cannot do that. */ rec_mtc(state, block, offset); return; } @@ -1957,15 +2081,23 @@ rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) jit_orr(tmp, tmp, tmp2); } + lightrec_free_reg(reg_cache, rt); + if (c.r.rd == 12 || c.r.rd == 13) { - jit_stxi_i(offsetof(struct lightrec_state, exit_flags), - LIGHTREC_REG_STATE, tmp); + to_end = jit_beqi(tmp, 0); - lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); - } + jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, target_cycle)); + jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE); + jit_movi(LIGHTREC_REG_CYCLE, 0); + jit_stxi_i(offsetof(struct lightrec_state, target_cycle), + LIGHTREC_REG_STATE, tmp2); + jit_stxi_i(offsetof(struct lightrec_state, current_cycle), + LIGHTREC_REG_STATE, tmp2); - lightrec_free_reg(reg_cache, rt); + + jit_patch(to_end); + } if (!op_flag_no_ds(block->opcode_list[offset].flags) && (c.r.rd == 12 || c.r.rd == 13)) { @@ -2022,15 +2154,14 @@ static unsigned int cp2c_s_offset(u8 reg) return cp2c_i_offset(reg) + is_big_endian() * 2; } -static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, - const struct block *block, u16 offset) +static void rec_cp2_do_mfc2(struct lightrec_cstate *state, + const struct block *block, u16 offset, + u8 reg, u8 out_reg) { struct regcache *reg_cache = state->reg_cache; - const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; const u32 zext_regs = 0x300f0080; u8 rt, tmp, tmp2, tmp3, out, flags; - u8 reg = c.r.rd == 15 ? 14 : c.r.rd; unsigned int i; _jit_name(block->_jit, __func__); @@ -2042,7 +2173,10 @@ static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, } flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT; - rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags); + rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); + + if (reg == 15) + reg = 14; switch (reg) { case 1: @@ -2099,6 +2233,14 @@ static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, lightrec_free_reg(reg_cache, rt); } +static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + const union code c = block->opcode_list[offset].c; + + rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt); +} + static void rec_cp2_basic_CFC2(struct lightrec_cstate *state, const struct block *block, u16 offset) { @@ -2128,20 +2270,19 @@ static void rec_cp2_basic_CFC2(struct lightrec_cstate *state, break; default: rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT); - jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd)); + jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd)); break; } lightrec_free_reg(reg_cache, rt); } -static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, - const struct block *block, u16 offset) +static void rec_cp2_do_mtc2(struct lightrec_cstate *state, + const struct block *block, u16 offset, + u8 reg, u8 in_reg) { struct regcache *reg_cache = state->reg_cache; - const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - jit_node_t *loop, *to_loop; u8 rt, tmp, tmp2, flags = 0; _jit_name(block->_jit, __func__); @@ -2152,15 +2293,15 @@ static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, return; } - if (c.r.rd == 31) + if (reg == 31) return; - if (c.r.rd == 30) + if (reg == 30) flags |= REG_EXT; - rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags); + rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags); - switch (c.r.rd) { + switch (reg) { case 15: tmp = lightrec_alloc_reg_temp(reg_cache, _jit); jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13)); @@ -2194,38 +2335,36 @@ static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, break; case 30: tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); /* if (rt < 0) rt = ~rt; */ jit_rshi(tmp, rt, 31); jit_xorr(tmp, rt, tmp); - /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */ - jit_lshi(tmp, tmp, 1); - jit_movi(tmp2, 33); - - /* Decrement tmp2 and right-shift the value by 1 until it equals zero */ - loop = jit_label(); - jit_subi(tmp2, tmp2, 1); - jit_rshi_u(tmp, tmp, 1); - to_loop = jit_bnei(tmp, 0); - - jit_patch_at(to_loop, loop); + /* Count leading zeros */ + jit_clzr(tmp, tmp); + if (__WORDSIZE != 32) + jit_subi(tmp, tmp, __WORDSIZE - 32); - jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2); - jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt); + jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp); lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); - break; + fallthrough; default: - jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt); + jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt); break; } lightrec_free_reg(reg_cache, rt); } +static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + const union code c = block->opcode_list[offset].c; + + rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt); +} + static void rec_cp2_basic_CTC2(struct lightrec_cstate *state, const struct block *block, u16 offset) { @@ -2334,24 +2473,56 @@ static void rec_meta_MOV(struct lightrec_cstate *state, const struct block *block, u16 offset) { struct regcache *reg_cache = state->reg_cache; - union code c = block->opcode_list[offset].c; + const struct opcode *op = &block->opcode_list[offset]; + union code c = op->c; jit_state_t *_jit = block->_jit; + bool unload_rd; u8 rs, rd; _jit_name(block->_jit, __func__); jit_note(__FILE__, __LINE__); - if (c.r.rs) - rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); - rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT); - if (c.r.rs == 0) - jit_movi(rd, 0); - else - jit_extr_i(rd, rs); + unload_rd = OPT_EARLY_UNLOAD + && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD; + + if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) { + /* The source register is not yet loaded - we can load its value + * from the register cache directly into the target register. */ + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); + + jit_ldxi_i(rd, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2)); + + lightrec_free_reg(reg_cache, rd); + } else if (unload_rd) { + /* If the destination register will be unloaded right after the + * MOV meta-opcode, we don't actually need to write any host + * register - we can just store the source register directly to + * the register cache, at the offset corresponding to the + * destination register. */ + lightrec_discard_reg_if_loaded(reg_cache, c.m.rd); + + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); + + jit_stxi_i(offsetof(struct lightrec_state, regs.gpr) + + (c.m.rd << 2), LIGHTREC_REG_STATE, rs); - if (c.r.rs) lightrec_free_reg(reg_cache, rs); - lightrec_free_reg(reg_cache, rd); + } else { + if (c.m.rs) + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); + + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); + + if (c.m.rs == 0) { + jit_movi(rd, 0); + } else { + jit_extr_i(rd, rs); + lightrec_free_reg(reg_cache, rs); + } + + lightrec_free_reg(reg_cache, rd); + } } static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state, @@ -2361,21 +2532,21 @@ static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state, struct regcache *reg_cache = state->reg_cache; union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rs, rt; + u8 rs, rd; _jit_name(block->_jit, __func__); jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); - rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); - if (c.i.op == OP_META_EXTC) - jit_extr_c(rt, rs); + if (c.m.op == OP_META_EXTC) + jit_extr_c(rd, rs); else - jit_extr_s(rt, rs); + jit_extr_s(rd, rs); lightrec_free_reg(reg_cache, rs); - lightrec_free_reg(reg_cache, rt); + lightrec_free_reg(reg_cache, rd); } static void rec_meta_MULT2(struct lightrec_cstate *state, @@ -2390,6 +2561,7 @@ static void rec_meta_MULT2(struct lightrec_cstate *state, u32 flags = block->opcode_list[offset].flags; bool is_signed = c.i.op == OP_META_MULT2; u8 rs, lo, hi, rflags = 0, hiflags = 0; + unsigned int i; if (!op_flag_no_hi(flags) && c.r.op < 32) { rflags = is_signed ? REG_EXT : REG_ZEXT; @@ -2401,28 +2573,38 @@ static void rec_meta_MULT2(struct lightrec_cstate *state, rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags); - if (!op_flag_no_lo(flags)) { - lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); - - if (c.r.op < 32) - jit_lshi(lo, rs, c.r.op); - else - jit_movi(lo, 0); + /* + * We must handle the case where one of the output registers is our rs + * input register. Thanksfully, computing LO/HI can be done in any + * order. Here, we make sure that the computation that overwrites the + * input register is always performed last. + */ + for (i = 0; i < 2; i++) { + if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) { + lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); + + if (c.r.op < 32) + jit_lshi(lo, rs, c.r.op); + else + jit_movi(lo, 0); - lightrec_free_reg(reg_cache, lo); - } + lightrec_free_reg(reg_cache, lo); + continue; + } - if (!op_flag_no_hi(flags)) { - hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags); + if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) { + hi = lightrec_alloc_reg_out(reg_cache, _jit, + reg_hi, hiflags); - if (c.r.op >= 32) - jit_lshi(hi, rs, c.r.op - 32); - else if (is_signed) - jit_rshi(hi, rs, 32 - c.r.op); - else - jit_rshi_u(hi, rs, 32 - c.r.op); + if (c.r.op >= 32) + jit_lshi(hi, rs, c.r.op - 32); + else if (is_signed) + jit_rshi(hi, rs, 32 - c.r.op); + else + jit_rshi_u(hi, rs, 32 - c.r.op); - lightrec_free_reg(reg_cache, hi); + lightrec_free_reg(reg_cache, hi); + } } lightrec_free_reg(reg_cache, rs); @@ -2431,6 +2613,29 @@ static void rec_meta_MULT2(struct lightrec_cstate *state, jit_note(__FILE__, __LINE__); } +static void rec_meta_COM(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rd, rs, flags; + + jit_note(__FILE__, __LINE__); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, 0); + + flags = lightrec_get_reg_in_flags(reg_cache, rs); + + lightrec_set_reg_out_flags(reg_cache, rd, + flags & REG_EXT); + + jit_comr(rd, rs); + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rd); +} + static const lightrec_rec_func_t rec_standard[64] = { SET_DEFAULT_ELM(rec_standard, unknown_opcode), [OP_SPECIAL] = rec_SPECIAL, @@ -2463,12 +2668,10 @@ static const lightrec_rec_func_t rec_standard[64] = { [OP_SWL] = rec_SWL, [OP_SW] = rec_SW, [OP_SWR] = rec_SWR, - [OP_LWC2] = rec_LWC2, - [OP_SWC2] = rec_SWC2, + [OP_LWC2] = rec_LW, + [OP_SWC2] = rec_SW, - [OP_META_MOV] = rec_meta_MOV, - [OP_META_EXTC] = rec_meta_EXTC_EXTS, - [OP_META_EXTS] = rec_meta_EXTC_EXTS, + [OP_META] = rec_META, [OP_META_MULT2] = rec_meta_MULT2, [OP_META_MULTU2] = rec_meta_MULT2, }; @@ -2530,6 +2733,14 @@ static const lightrec_rec_func_t rec_cp2_basic[64] = { [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2, }; +static const lightrec_rec_func_t rec_meta[64] = { + SET_DEFAULT_ELM(rec_meta, unknown_opcode), + [OP_META_MOV] = rec_meta_MOV, + [OP_META_EXTC] = rec_meta_EXTC_EXTS, + [OP_META_EXTS] = rec_meta_EXTC_EXTS, + [OP_META_COM] = rec_meta_COM, +}; + static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset) { @@ -2583,6 +2794,18 @@ static void rec_CP2(struct lightrec_cstate *state, rec_CP(state, block, offset); } +static void rec_META(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + union code c = block->opcode_list[offset].c; + lightrec_rec_func_t f = rec_meta[c.m.op]; + + if (!HAS_DEFAULT_ELM && unlikely(!f)) + unknown_opcode(state, block, offset); + else + (*f)(state, block, offset); +} + void lightrec_rec_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset) { @@ -2622,4 +2845,6 @@ void lightrec_rec_opcode(struct lightrec_cstate *state, lightrec_do_early_unload(state, block, unload_offset); } + + state->no_load_delay = false; }