X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=deps%2Flightrec%2Femitter.c;h=fd2893560962ab0f3d8da6308767b118546e49c8;hb=02487de7ff9fcbb6d7d692a6b3ae6e6539708abc;hp=b09dc94927b810f20ac125ef52adfa2067febc62;hpb=0adc619b5a5705d16cc6caea64951776ababbb96;p=pcsx_rearmed.git diff --git a/deps/lightrec/emitter.c b/deps/lightrec/emitter.c index b09dc949..fd289356 100644 --- a/deps/lightrec/emitter.c +++ b/deps/lightrec/emitter.c @@ -1,61 +1,50 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later /* - * Copyright (C) 2014-2020 Paul Cercueil - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. + * Copyright (C) 2014-2021 Paul Cercueil */ #include "blockcache.h" #include "debug.h" #include "disassembler.h" #include "emitter.h" +#include "lightning-wrapper.h" #include "optimizer.h" #include "regcache.h" -#include #include #include -typedef void (*lightrec_rec_func_t)(const struct block *, - const struct opcode *, u32); +typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16); /* Forward declarations */ -static void rec_SPECIAL(const struct block *block, - const struct opcode *op, u32 pc); -static void rec_REGIMM(const struct block *block, - const struct opcode *op, u32 pc); -static void rec_CP0(const struct block *block, const struct opcode *op, u32 pc); -static void rec_CP2(const struct block *block, const struct opcode *op, u32 pc); +static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset); +static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset); +static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset); +static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset); - -static void unknown_opcode(const struct block *block, - const struct opcode *op, u32 pc) +static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset) { - pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n", op->opcode, pc); + pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n", + block->opcode_list[offset].c.opcode, + block->pc + (offset << 2)); } -static void lightrec_emit_end_of_block(const struct block *block, - const struct opcode *op, u32 pc, +static void lightrec_emit_end_of_block(struct lightrec_cstate *state, + const struct block *block, u16 offset, s8 reg_new_pc, u32 imm, u8 ra_reg, u32 link, bool update_cycles) { - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; u32 cycles = state->cycles; jit_state_t *_jit = block->_jit; + const struct opcode *op = &block->opcode_list[offset], + *next = &block->opcode_list[offset + 1]; jit_note(__FILE__, __LINE__); if (link) { /* Update the $ra register */ - u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg); + u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0); jit_movi(link_reg, link); lightrec_free_reg(reg_cache, link_reg); } @@ -69,11 +58,11 @@ static void lightrec_emit_end_of_block(const struct block *block, if (has_delay_slot(op->c) && !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) { - cycles += lightrec_cycles_of_opcode(op->next->c); + cycles += lightrec_cycles_of_opcode(next->c); /* Recompile the delay slot */ - if (op->next->c.opcode) - lightrec_rec_opcode(block, op->next, pc + 4); + if (next->c.opcode) + lightrec_rec_opcode(state, block, offset + 1); } /* Store back remaining registers */ @@ -86,91 +75,109 @@ static void lightrec_emit_end_of_block(const struct block *block, pr_debug("EOB: %u cycles\n", cycles); } - if (op->next && ((op->flags & LIGHTREC_NO_DS) || op->next->next)) - state->branches[state->nb_branches++] = jit_jmpi(); + if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1) + state->branches[state->nb_branches++] = jit_b(); } -void lightrec_emit_eob(const struct block *block, - const struct opcode *op, u32 pc) +void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block, + u16 offset, bool after_op) { - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; + union code c = block->opcode_list[offset].c; + u32 cycles = state->cycles; + + if (!after_op) + cycles -= lightrec_cycles_of_opcode(c); lightrec_storeback_regs(reg_cache, _jit); - jit_movi(JIT_V0, pc); - jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, - state->cycles - lightrec_cycles_of_opcode(op->c)); + jit_movi(JIT_V0, block->pc + (offset << 2)); + jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); - state->branches[state->nb_branches++] = jit_jmpi(); + state->branches[state->nb_branches++] = jit_b(); } -static void rec_special_JR(const struct block *block, - const struct opcode *op, u32 pc) +static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; - u8 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0); + const struct opcode *op = &block->opcode_list[offset]; + u8 rs; - _jit_name(block->_jit, __func__); + rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0); lightrec_lock_reg(reg_cache, _jit, rs); - lightrec_emit_end_of_block(block, op, pc, rs, 0, 31, 0, true); + + return rs; } -static void rec_special_JALR(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; - jit_state_t *_jit = block->_jit; - u8 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0); + u8 rs = get_jr_jalr_reg(state, block, offset); _jit_name(block->_jit, __func__); - lightrec_lock_reg(reg_cache, _jit, rs); - lightrec_emit_end_of_block(block, op, pc, rs, 0, op->r.rd, pc + 8, true); + lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true); } -static void rec_J(const struct block *block, const struct opcode *op, u32 pc) +static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset) { + u8 rs = get_jr_jalr_reg(state, block, offset); + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - lightrec_emit_end_of_block(block, op, pc, -1, - (pc & 0xf0000000) | (op->j.imm << 2), 31, 0, true); + lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd, + get_branch_pc(block, offset, 2), true); } -static void rec_JAL(const struct block *block, const struct opcode *op, u32 pc) +static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + + _jit_name(block->_jit, __func__); + lightrec_emit_end_of_block(state, block, offset, -1, + (block->pc & 0xf0000000) | (c.j.imm << 2), + 31, 0, true); +} + +static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset) +{ + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - lightrec_emit_end_of_block(block, op, pc, -1, - (pc & 0xf0000000) | (op->j.imm << 2), - 31, pc + 8, true); + lightrec_emit_end_of_block(state, block, offset, -1, + (block->pc & 0xf0000000) | (c.j.imm << 2), + 31, get_branch_pc(block, offset, 2), true); } -static void rec_b(const struct block *block, const struct opcode *op, u32 pc, +static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset, jit_code_t code, u32 link, bool unconditional, bool bz) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; struct native_register *regs_backup; jit_state_t *_jit = block->_jit; struct lightrec_branch *branch; + const struct opcode *op = &block->opcode_list[offset], + *next = &block->opcode_list[offset + 1]; jit_node_t *addr; u8 link_reg; - u32 offset, cycles = block->state->cycles; + u32 target_offset, cycles = state->cycles; bool is_forward = (s16)op->i.imm >= -1; + u32 next_pc; jit_note(__FILE__, __LINE__); if (!(op->flags & LIGHTREC_NO_DS)) - cycles += lightrec_cycles_of_opcode(op->next->c); + cycles += lightrec_cycles_of_opcode(next->c); - block->state->cycles = 0; + state->cycles = 0; if (cycles) jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); if (!unconditional) { - u8 rs = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->i.rs), - rt = bz ? 0 : lightrec_alloc_reg_in_ext(reg_cache, - _jit, op->i.rt); + u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT), + rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache, + _jit, op->i.rt, REG_EXT); /* Generate the branch opcode */ addr = jit_new_node_pww(code, NULL, rs, rt); @@ -180,15 +187,15 @@ static void rec_b(const struct block *block, const struct opcode *op, u32 pc, } if (op->flags & LIGHTREC_LOCAL_BRANCH) { - if (op->next && !(op->flags & LIGHTREC_NO_DS)) { + if (next && !(op->flags & LIGHTREC_NO_DS)) { /* Recompile the delay slot */ - if (op->next->opcode) - lightrec_rec_opcode(block, op->next, pc + 4); + if (next->opcode) + lightrec_rec_opcode(state, block, offset + 1); } if (link) { /* Update the $ra register */ - link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31); + link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0); jit_movi(link_reg, link); lightrec_free_reg(reg_cache, link_reg); } @@ -196,21 +203,23 @@ static void rec_b(const struct block *block, const struct opcode *op, u32 pc, /* Store back remaining registers */ lightrec_storeback_regs(reg_cache, _jit); - offset = op->offset + 1 + (s16)op->i.imm; - pr_debug("Adding local branch to offset 0x%x\n", offset << 2); - branch = &block->state->local_branches[ - block->state->nb_local_branches++]; + target_offset = offset + 1 + (s16)op->i.imm + - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS)); + pr_debug("Adding local branch to offset 0x%x\n", + target_offset << 2); + branch = &state->local_branches[ + state->nb_local_branches++]; - branch->target = offset; + branch->target = target_offset; if (is_forward) - branch->branch = jit_jmpi(); + branch->branch = jit_b(); else branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0); } if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) { - lightrec_emit_end_of_block(block, op, pc, -1, - pc + 4 + ((s16)op->i.imm << 2), + next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm); + lightrec_emit_end_of_block(state, block, offset, -1, next_pc, 31, link, false); } @@ -220,105 +229,127 @@ static void rec_b(const struct block *block, const struct opcode *op, u32 pc, if (bz && link) { /* Update the $ra register */ - link_reg = lightrec_alloc_reg_out_ext(reg_cache, - _jit, 31); + link_reg = lightrec_alloc_reg_out(reg_cache, _jit, + 31, REG_EXT); jit_movi(link_reg, (s32)link); lightrec_free_reg(reg_cache, link_reg); } - if (!(op->flags & LIGHTREC_NO_DS) && op->next->opcode) - lightrec_rec_opcode(block, op->next, pc + 4); + if (!(op->flags & LIGHTREC_NO_DS) && next->opcode) + lightrec_rec_opcode(state, block, offset + 1); } } -static void rec_BNE(const struct block *block, const struct opcode *op, u32 pc) +static void rec_BNE(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_beqr, 0, false, false); + + if (c.i.rt == 0) + rec_b(state, block, offset, jit_code_beqi, 0, false, true); + else + rec_b(state, block, offset, jit_code_beqr, 0, false, false); } -static void rec_BEQ(const struct block *block, const struct opcode *op, u32 pc) +static void rec_BEQ(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_bner, 0, - op->i.rs == op->i.rt, false); + + if (c.i.rt == 0) + rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true); + else + rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false); } -static void rec_BLEZ(const struct block *block, const struct opcode *op, u32 pc) +static void rec_BLEZ(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_bgti, 0, op->i.rs == 0, true); + rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true); } -static void rec_BGTZ(const struct block *block, const struct opcode *op, u32 pc) +static void rec_BGTZ(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_blei, 0, false, true); + rec_b(state, block, offset, jit_code_blei, 0, false, true); } -static void rec_regimm_BLTZ(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_regimm_BLTZ(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_bgei, 0, false, true); + rec_b(state, block, offset, jit_code_bgei, 0, false, true); } -static void rec_regimm_BLTZAL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_regimm_BLTZAL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_bgei, pc + 8, false, true); + rec_b(state, block, offset, jit_code_bgei, + get_branch_pc(block, offset, 2), false, true); } -static void rec_regimm_BGEZ(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_regimm_BGEZ(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_blti, 0, !op->i.rs, true); + rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true); } -static void rec_regimm_BGEZAL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_regimm_BGEZAL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + const struct opcode *op = &block->opcode_list[offset]; _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_blti, pc + 8, !op->i.rs, true); + rec_b(state, block, offset, jit_code_blti, + get_branch_pc(block, offset, 2), + !op->i.rs, true); } -static void rec_alu_imm(const struct block *block, const struct opcode *op, - jit_code_t code, bool sign_extend) +static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code, bool slti) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rs, rt; + u8 rs, rt, out_flags = REG_EXT; + + if (slti) + out_flags |= REG_ZEXT; jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->i.rs); - rt = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->i.rt); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags); - if (sign_extend) - jit_new_node_www(code, rt, rs, (s32)(s16) op->i.imm); - else - jit_new_node_www(code, rt, rs, (u32)(u16) op->i.imm); + jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm); lightrec_free_reg(reg_cache, rs); lightrec_free_reg(reg_cache, rt); } -static void rec_alu_special(const struct block *block, const struct opcode *op, - jit_code_t code, bool out_ext) +static void rec_alu_special(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code, bool out_ext) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; u8 rd, rt, rs; jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rs); - rt = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rt); - - if (out_ext) - rd = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->r.rd); - else - rd = lightrec_alloc_reg_out(reg_cache, _jit, op->r.rd); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, + out_ext ? REG_EXT | REG_ZEXT : 0); jit_new_node_www(code, rd, rs, rt); @@ -327,539 +358,851 @@ static void rec_alu_special(const struct block *block, const struct opcode *op, lightrec_free_reg(reg_cache, rd); } -static void rec_alu_shiftv(const struct block *block, - const struct opcode *op, jit_code_t code) +static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rd, rt, rs, temp; + u8 rd, rt, rs, temp, flags = 0; jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rs); - temp = lightrec_alloc_reg_temp(reg_cache, _jit); - - if (code == jit_code_rshr) { - rt = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rt); - rd = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->r.rd); - } else { - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rt); - rd = lightrec_alloc_reg_out(reg_cache, _jit, op->r.rd); - } + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); - jit_andi(temp, rs, 0x1f); + if (code == jit_code_rshr) + flags = REG_EXT; + else if (code == jit_code_rshr_u) + flags = REG_ZEXT; -#if __WORDSIZE == 64 - if (code == jit_code_rshr_u) { - jit_extr_ui(rd, rt); - jit_new_node_www(code, rd, rd, temp); - } -#endif + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags); - if (__WORDSIZE == 32 || code != jit_code_rshr_u) + if (rs != rd && rt != rd) { + jit_andi(rd, rs, 0x1f); + jit_new_node_www(code, rd, rt, rd); + } else { + temp = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_andi(temp, rs, 0x1f); jit_new_node_www(code, rd, rt, temp); + lightrec_free_reg(reg_cache, temp); + } lightrec_free_reg(reg_cache, rs); - lightrec_free_reg(reg_cache, temp); lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, rd); } -static void rec_ADDIU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_movi(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u16 flags = REG_EXT; + u8 rt; + + if (!(c.i.imm & 0x8000)) + flags |= REG_ZEXT; + + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); + + jit_movi(rt, (s32)(s16) c.i.imm); + + lightrec_free_reg(reg_cache, rt); +} + +static void rec_ADDIU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_addi, true); + + if (block->opcode_list[offset].c.i.rs) + rec_alu_imm(state, block, offset, jit_code_addi, false); + else + rec_movi(state, block, offset); } -static void rec_ADDI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_ADDI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { /* TODO: Handle the exception? */ _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_addi, true); + rec_ADDIU(state, block, offset); } -static void rec_SLTIU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_SLTIU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_lti_u, true); + rec_alu_imm(state, block, offset, jit_code_lti_u, true); } -static void rec_SLTI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SLTI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_lti, true); + rec_alu_imm(state, block, offset, jit_code_lti, true); } -static void rec_ANDI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_ANDI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; u8 rs, rt; _jit_name(block->_jit, __func__); jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs); - rt = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->i.rt); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, + REG_EXT | REG_ZEXT); /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically * casts to uint8_t / uint16_t. */ - if (op->i.imm == 0xff) + if (c.i.imm == 0xff) jit_extr_uc(rt, rs); - else if (op->i.imm == 0xffff) + else if (c.i.imm == 0xffff) jit_extr_us(rt, rs); else - jit_andi(rt, rs, (u32)(u16) op->i.imm); + jit_andi(rt, rs, (u32)(u16) c.i.imm); lightrec_free_reg(reg_cache, rs); lightrec_free_reg(reg_cache, rt); } -static void rec_ORI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code) +{ + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rs, rt, flags; + + jit_note(__FILE__, __LINE__); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0); + + flags = lightrec_get_reg_in_flags(reg_cache, rs); + lightrec_set_reg_out_flags(reg_cache, rt, flags); + + jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm); + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); +} + + +static void rec_ORI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_ori, false); + rec_alu_or_xor(state, block, offset, jit_code_ori); } -static void rec_XORI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_XORI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_imm(block, op, jit_code_xori, false); + rec_alu_or_xor(state, block, offset, jit_code_xori); } -static void rec_LUI(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LUI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rt; + u8 rt, flags = REG_EXT; jit_name(__func__); jit_note(__FILE__, __LINE__); - rt = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->i.rt); - jit_movi(rt, (s32)(op->i.imm << 16)); + if (!(c.i.imm & BIT(15))) + flags |= REG_ZEXT; + + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); + + jit_movi(rt, (s32)(c.i.imm << 16)); lightrec_free_reg(reg_cache, rt); } -static void rec_special_ADDU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_ADDU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_addr, false); + rec_alu_special(state, block, offset, jit_code_addr, false); } -static void rec_special_ADD(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_ADD(struct lightrec_cstate *state, + const struct block *block, u16 offset) { /* TODO: Handle the exception? */ _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_addr, false); + rec_alu_special(state, block, offset, jit_code_addr, false); } -static void rec_special_SUBU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SUBU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_subr, false); + rec_alu_special(state, block, offset, jit_code_subr, false); } -static void rec_special_SUB(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SUB(struct lightrec_cstate *state, + const struct block *block, u16 offset) { /* TODO: Handle the exception? */ _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_subr, false); + rec_alu_special(state, block, offset, jit_code_subr, false); } -static void rec_special_AND(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_AND(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; + _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_andr, false); + jit_note(__FILE__, __LINE__); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0); + + flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); + flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); + + /* Z(rd) = Z(rs) | Z(rt) */ + flags_rd = REG_ZEXT & (flags_rs | flags_rt); + + /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */ + if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) || + ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) || + (REG_EXT & flags_rs & flags_rt)) + flags_rd |= REG_EXT; + + lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); + + jit_andr(rd, rs, rt); + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); + lightrec_free_reg(reg_cache, rd); +} + +static void rec_special_or_nor(struct lightrec_cstate *state, + const struct block *block, u16 offset, bool nor) +{ + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0; + + jit_note(__FILE__, __LINE__); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0); + + flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); + flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); + + /* or: Z(rd) = Z(rs) & Z(rt) + * nor: Z(rd) = 0 */ + if (!nor) + flags_rd = REG_ZEXT & flags_rs & flags_rt; + + /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */ + if ((REG_EXT & flags_rs & flags_rt) || + (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) || + (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT)) + flags_rd |= REG_EXT; + + lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); + + jit_orr(rd, rs, rt); + + if (nor) + jit_comr(rd, rd); + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); + lightrec_free_reg(reg_cache, rd); } -static void rec_special_OR(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_OR(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_orr, false); + rec_special_or_nor(state, block, offset, false); } -static void rec_special_XOR(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_NOR(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_xorr, false); + rec_special_or_nor(state, block, offset, true); } -static void rec_special_NOR(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_XOR(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rd; + u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; - jit_name(__func__); - rec_alu_special(block, op, jit_code_orr, false); - rd = lightrec_alloc_reg_out(reg_cache, _jit, op->r.rd); + _jit_name(block->_jit, __func__); - jit_comr(rd, rd); + jit_note(__FILE__, __LINE__); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0); + + flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); + flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); + + /* Z(rd) = Z(rs) & Z(rt) */ + flags_rd = REG_ZEXT & flags_rs & flags_rt; + + /* E(rd) = E(rs) & E(rt) */ + flags_rd |= REG_EXT & flags_rs & flags_rt; + lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); + + jit_xorr(rd, rs, rt); + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, rd); } -static void rec_special_SLTU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SLTU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_ltr_u, true); + rec_alu_special(state, block, offset, jit_code_ltr_u, true); } -static void rec_special_SLT(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SLT(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_special(block, op, jit_code_ltr, true); + rec_alu_special(state, block, offset, jit_code_ltr, true); } -static void rec_special_SLLV(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SLLV(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shiftv(block, op, jit_code_lshr); + rec_alu_shiftv(state, block, offset, jit_code_lshr); } -static void rec_special_SRLV(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SRLV(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shiftv(block, op, jit_code_rshr_u); + rec_alu_shiftv(state, block, offset, jit_code_rshr_u); } -static void rec_special_SRAV(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SRAV(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shiftv(block, op, jit_code_rshr); + rec_alu_shiftv(state, block, offset, jit_code_rshr); } -static void rec_alu_shift(const struct block *block, - const struct opcode *op, jit_code_t code) +static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 rd, rt; + u8 rd, rt, flags = 0; jit_note(__FILE__, __LINE__); - if (code == jit_code_rshi) { - rt = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rt); - rd = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->r.rd); - } else { - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rt); - rd = lightrec_alloc_reg_out(reg_cache, _jit, op->r.rd); - } + if (code == jit_code_rshi) + flags = REG_EXT; + else if (code == jit_code_rshi_u) + flags = REG_ZEXT; -#if __WORDSIZE == 64 - if (code == jit_code_rshi_u) { - jit_extr_ui(rd, rt); - jit_new_node_www(code, rd, rd, op->r.imm); - } -#endif - if (__WORDSIZE == 32 || code != jit_code_rshi_u) - jit_new_node_www(code, rd, rt, op->r.imm); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags); + + /* Input reg is zero-extended, if we SRL at least by one bit, we know + * the output reg will be both zero-extended and sign-extended. */ + if (code == jit_code_rshi_u && c.r.imm) + flags |= REG_EXT; + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags); + + jit_new_node_www(code, rd, rt, c.r.imm); lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, rd); } -static void rec_special_SLL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SLL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shift(block, op, jit_code_lshi); + rec_alu_shift(state, block, offset, jit_code_lshi); } -static void rec_special_SRL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SRL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shift(block, op, jit_code_rshi_u); + rec_alu_shift(state, block, offset, jit_code_rshi_u); } -static void rec_special_SRA(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SRA(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_shift(block, op, jit_code_rshi); + rec_alu_shift(state, block, offset, jit_code_rshi); } -static void rec_alu_mult(const struct block *block, - const struct opcode *op, bool is_signed) +static void rec_alu_mult(struct lightrec_cstate *state, + const struct block *block, u16 offset, bool is_signed) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + u16 flags = block->opcode_list[offset].flags; + u8 reg_lo = get_mult_div_lo(c); + u8 reg_hi = get_mult_div_hi(c); jit_state_t *_jit = block->_jit; - u8 lo, hi, rs, rt; + u8 lo, hi, rs, rt, rflags = 0; jit_note(__FILE__, __LINE__); - lo = lightrec_alloc_reg_out(reg_cache, _jit, REG_LO); - if (!(op->flags & LIGHTREC_MULT32)) - hi = lightrec_alloc_reg_out_ext(reg_cache, _jit, REG_HI); - else if (__WORDSIZE == 64) - hi = lightrec_alloc_reg_temp(reg_cache, _jit); - - if (__WORDSIZE == 32 || !is_signed) { - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rs); - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rt); + if (is_signed) + rflags = REG_EXT; + else + rflags = REG_ZEXT; + + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); + + if (!(flags & LIGHTREC_NO_LO)) + lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); + else if (__WORDSIZE == 32) + lo = lightrec_alloc_reg_temp(reg_cache, _jit); + + if (!(flags & LIGHTREC_NO_HI)) + hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT); + + if (__WORDSIZE == 32) { + /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit + * operation if the MULT was detected a 32-bit only. */ + if (!(flags & LIGHTREC_NO_HI)) { + if (is_signed) + jit_qmulr(lo, hi, rs, rt); + else + jit_qmulr_u(lo, hi, rs, rt); + } else { + jit_mulr(lo, rs, rt); + } } else { - rs = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rs); - rt = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rt); - } + /* On 64-bit systems, do a 64*64->64 bit operation. */ + if (flags & LIGHTREC_NO_LO) { + jit_mulr(hi, rs, rt); + jit_rshi(hi, hi, 32); + } else { + jit_mulr(lo, rs, rt); -#if __WORDSIZE == 32 - /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit - * operation if the MULT was detected a 32-bit only. */ - if (!(op->flags & LIGHTREC_MULT32)) { - if (is_signed) - jit_qmulr(lo, hi, rs, rt); - else - jit_qmulr_u(lo, hi, rs, rt); - } else { - jit_mulr(lo, rs, rt); - } -#else - /* On 64-bit systems, do a 64*64->64 bit operation. - * The input registers must be 32 bits, so we first sign-extend (if - * mult) or clear (if multu) the input registers. */ - if (is_signed) { - jit_mulr(lo, rs, rt); - } else { - jit_extr_ui(lo, rt); - jit_extr_ui(hi, rs); - jit_mulr(lo, hi, lo); + /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */ + if (!(flags & LIGHTREC_NO_HI)) + jit_rshi(hi, lo, 32); + } } - /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */ - if (!(op->flags & LIGHTREC_MULT32)) - jit_rshi(hi, lo, 32); -#endif - lightrec_free_reg(reg_cache, rs); lightrec_free_reg(reg_cache, rt); - lightrec_free_reg(reg_cache, lo); - if (__WORDSIZE == 64 || !(op->flags & LIGHTREC_MULT32)) + if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32) + lightrec_free_reg(reg_cache, lo); + if (!(flags & LIGHTREC_NO_HI)) lightrec_free_reg(reg_cache, hi); } -static void rec_alu_div(const struct block *block, - const struct opcode *op, bool is_signed) +static void rec_alu_div(struct lightrec_cstate *state, + const struct block *block, u16 offset, bool is_signed) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + u16 flags = block->opcode_list[offset].flags; + bool no_check = flags & LIGHTREC_NO_DIV_CHECK; + u8 reg_lo = get_mult_div_lo(c); + u8 reg_hi = get_mult_div_hi(c); jit_state_t *_jit = block->_jit; jit_node_t *branch, *to_end; - u8 lo, hi, rs, rt; + u8 lo = 0, hi = 0, rs, rt, rflags = 0; jit_note(__FILE__, __LINE__); - lo = lightrec_alloc_reg_out(reg_cache, _jit, REG_LO); - hi = lightrec_alloc_reg_out(reg_cache, _jit, REG_HI); - if (__WORDSIZE == 32 || !is_signed) { - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rs); - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->r.rt); - } else { - rs = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rs); - rt = lightrec_alloc_reg_in_ext(reg_cache, _jit, op->r.rt); - } + if (is_signed) + rflags = REG_EXT; + else + rflags = REG_ZEXT; + + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); + + if (!(flags & LIGHTREC_NO_LO)) + lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); + + if (!(flags & LIGHTREC_NO_HI)) + hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0); /* Jump to special handler if dividing by zero */ - branch = jit_beqi(rt, 0); + if (!no_check) + branch = jit_beqi(rt, 0); -#if __WORDSIZE == 32 - if (is_signed) - jit_qdivr(lo, hi, rs, rt); - else - jit_qdivr_u(lo, hi, rs, rt); -#else - /* On 64-bit systems, the input registers must be 32 bits, so we first sign-extend - * (if div) or clear (if divu) the input registers. */ - if (is_signed) { - jit_qdivr(lo, hi, rs, rt); + if (flags & LIGHTREC_NO_LO) { + if (is_signed) + jit_remr(hi, rs, rt); + else + jit_remr_u(hi, rs, rt); + } else if (flags & LIGHTREC_NO_HI) { + if (is_signed) + jit_divr(lo, rs, rt); + else + jit_divr_u(lo, rs, rt); } else { - jit_extr_ui(lo, rt); - jit_extr_ui(hi, rs); - jit_qdivr_u(lo, hi, hi, lo); + if (is_signed) + jit_qdivr(lo, hi, rs, rt); + else + jit_qdivr_u(lo, hi, rs, rt); } -#endif - /* Jump above the div-by-zero handler */ - to_end = jit_jmpi(); + if (!no_check) { + /* Jump above the div-by-zero handler */ + to_end = jit_b(); - jit_patch(branch); + jit_patch(branch); - if (is_signed) { - jit_lti(lo, rs, 0); - jit_lshi(lo, lo, 1); - jit_subi(lo, lo, 1); - } else { - jit_movi(lo, 0xffffffff); - } + if (!(flags & LIGHTREC_NO_LO)) { + if (is_signed) { + jit_lti(lo, rs, 0); + jit_lshi(lo, lo, 1); + jit_subi(lo, lo, 1); + } else { + jit_movi(lo, 0xffffffff); + } + } - jit_movr(hi, rs); + if (!(flags & LIGHTREC_NO_HI)) + jit_movr(hi, rs); - jit_patch(to_end); + jit_patch(to_end); + } lightrec_free_reg(reg_cache, rs); lightrec_free_reg(reg_cache, rt); - lightrec_free_reg(reg_cache, lo); - lightrec_free_reg(reg_cache, hi); + + if (!(flags & LIGHTREC_NO_LO)) + lightrec_free_reg(reg_cache, lo); + + if (!(flags & LIGHTREC_NO_HI)) + lightrec_free_reg(reg_cache, hi); } -static void rec_special_MULT(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MULT(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_mult(block, op, true); + rec_alu_mult(state, block, offset, true); } -static void rec_special_MULTU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MULTU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_mult(block, op, false); + rec_alu_mult(state, block, offset, false); } -static void rec_special_DIV(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_DIV(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_div(block, op, true); + rec_alu_div(state, block, offset, true); } -static void rec_special_DIVU(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_DIVU(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_alu_div(block, op, false); + rec_alu_div(state, block, offset, false); } -static void rec_alu_mv_lo_hi(const struct block *block, u8 dst, u8 src) +static void rec_alu_mv_lo_hi(struct lightrec_cstate *state, + const struct block *block, u8 dst, u8 src) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; jit_note(__FILE__, __LINE__); - src = lightrec_alloc_reg_in(reg_cache, _jit, src); - dst = lightrec_alloc_reg_out_ext(reg_cache, _jit, dst); + src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0); + dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT); -#if __WORDSIZE == 32 - jit_movr(dst, src); -#else jit_extr_i(dst, src); -#endif lightrec_free_reg(reg_cache, src); lightrec_free_reg(reg_cache, dst); } -static void rec_special_MFHI(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MFHI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_alu_mv_lo_hi(block, op->r.rd, REG_HI); + rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI); } -static void rec_special_MTHI(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MTHI(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_alu_mv_lo_hi(block, REG_HI, op->r.rs); + rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs); } -static void rec_special_MFLO(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MFLO(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_alu_mv_lo_hi(block, op->r.rd, REG_LO); + rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO); } -static void rec_special_MTLO(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_MTLO(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + union code c = block->opcode_list[offset].c; + _jit_name(block->_jit, __func__); - rec_alu_mv_lo_hi(block, REG_LO, op->r.rs); + rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs); +} + +static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block, + u32 arg, bool with_arg, enum c_wrappers wrapper) +{ + struct regcache *reg_cache = state->reg_cache; + jit_state_t *_jit = block->_jit; + u8 tmp, tmp2; + + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_ldxi(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, wrappers_eps[wrapper])); + + if (with_arg) { + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_movi(tmp2, arg); + + jit_stxi_i(offsetof(struct lightrec_state, c_wrapper_arg), + LIGHTREC_REG_STATE, tmp2); + + lightrec_free_reg(reg_cache, tmp2); + } + + lightrec_regcache_mark_live(reg_cache, _jit); + jit_callr(tmp); + + lightrec_free_reg(reg_cache, tmp); + lightrec_regcache_mark_live(reg_cache, _jit); } -static void rec_io(const struct block *block, const struct opcode *op, +static void rec_io(struct lightrec_cstate *state, + const struct block *block, u16 offset, bool load_rt, bool read_rt) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; - bool is_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO); - u32 offset; - u8 tmp, tmp2, tmp3; + union code c = block->opcode_list[offset].c; + u16 flags = block->opcode_list[offset].flags; + bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags); + u32 lut_entry; jit_note(__FILE__, __LINE__); - tmp = lightrec_alloc_reg(reg_cache, _jit, JIT_R0); + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); + + if (read_rt && likely(c.i.rt)) + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); + else if (load_rt) + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); if (is_tagged) { - offset = offsetof(struct lightrec_state, rw_func); + call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW); } else { - tmp3 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1); - offset = offsetof(struct lightrec_state, rw_generic_func); + lut_entry = lightrec_get_lut_entry(block); + call_to_c_wrapper(state, block, (lut_entry << 16) | offset, + true, C_WRAPPER_RW_GENERIC); } +} - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); - jit_ldxi(tmp2, LIGHTREC_REG_STATE, offset); +static u32 rec_ram_mask(struct lightrec_state *state) +{ + return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1; +} - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, false); +static void rec_store_memory(struct lightrec_cstate *cstate, + const struct block *block, + u16 offset, jit_code_t code, + jit_code_t swap_code, + uintptr_t addr_offset, u32 addr_mask, + bool invalidate) +{ + const struct lightrec_state *state = cstate->state; + struct regcache *reg_cache = cstate->reg_cache; + struct opcode *op = &block->opcode_list[offset]; + jit_state_t *_jit = block->_jit; + union code c = op->c; + u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2; + s16 imm = (s16)c.i.imm; + s32 simm = (s32)imm << (1 - lut_is_32bit(state)); + s32 lut_offt = offsetof(struct lightrec_state, code_lut); + bool no_mask = op->flags & LIGHTREC_NO_MASK; + bool add_imm = c.i.imm && + ((!state->mirrors_mapped && !no_mask) || (invalidate && + ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt)))); + bool need_tmp = !no_mask || addr_offset || add_imm; + bool need_tmp2 = addr_offset || invalidate; + + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + if (need_tmp) + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + + addr_reg = rs; + + if (add_imm) { + jit_addi(tmp, addr_reg, (s16)c.i.imm); + addr_reg = tmp; + imm = 0; + } else if (simm) { + lut_offt += simm; + } - if (read_rt && likely(op->i.rt)) - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true); - else if (load_rt) - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, false); + if (!no_mask) { + jit_andi(tmp, addr_reg, addr_mask); + addr_reg = tmp; + } - if (is_tagged) { - jit_movi(tmp, op->opcode); + if (need_tmp2) + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + + if (addr_offset) { + jit_addi(tmp2, addr_reg, addr_offset); + addr_reg2 = tmp2; } else { - jit_movi(tmp, (uintptr_t)op); - jit_movi(tmp3, (uintptr_t)block); + addr_reg2 = addr_reg; } - jit_callr(tmp2); + if (is_big_endian() && swap_code && c.i.rt) { + tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); + + jit_new_node_ww(swap_code, tmp3, rt); + jit_new_node_www(code, imm, addr_reg2, tmp3); - lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); - if (!is_tagged) lightrec_free_reg(reg_cache, tmp3); - lightrec_regcache_mark_live(reg_cache, _jit); + } else { + jit_new_node_www(code, imm, addr_reg2, rt); + } + + lightrec_free_reg(reg_cache, rt); + + if (invalidate) { + tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); + + if (c.i.op != OP_SW) { + jit_andi(tmp2, addr_reg, ~3); + addr_reg = tmp2; + } + + if (!lut_is_32bit(state)) { + jit_lshi(tmp2, addr_reg, 1); + addr_reg = tmp2; + } + + if (addr_reg == rs && c.i.rs == 0) { + addr_reg = LIGHTREC_REG_STATE; + } else { + jit_addr(tmp2, addr_reg, LIGHTREC_REG_STATE); + addr_reg = tmp2; + } + + if (lut_is_32bit(state)) + jit_stxi_i(lut_offt, addr_reg, tmp3); + else + jit_stxi(lut_offt, addr_reg, tmp3); + + lightrec_free_reg(reg_cache, tmp3); + } + + if (need_tmp2) + lightrec_free_reg(reg_cache, tmp2); + if (need_tmp) + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, rs); } -static void rec_store_direct_no_invalidate(const struct block *block, - const struct opcode *op, - jit_code_t code) +static void rec_store_ram(struct lightrec_cstate *cstate, + const struct block *block, + u16 offset, jit_code_t code, + jit_code_t swap_code, bool invalidate) { - struct lightrec_state *state = block->state; - struct regcache *reg_cache = state->reg_cache; + struct lightrec_state *state = cstate->state; + + _jit_note(block->_jit, __FILE__, __LINE__); + + return rec_store_memory(cstate, block, offset, code, swap_code, + state->offset_ram, rec_ram_mask(state), + invalidate); +} + +static void rec_store_scratch(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code) +{ + _jit_note(block->_jit, __FILE__, __LINE__); + + return rec_store_memory(cstate, block, offset, code, swap_code, + cstate->state->offset_scratch, + 0x1fffffff, false); +} + +static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate, + const struct block *block, + u16 offset, jit_code_t code, + jit_code_t swap_code) +{ + struct lightrec_state *state = cstate->state; + struct regcache *reg_cache = cstate->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_end; u8 tmp, tmp2, rs, rt; s16 imm; jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + + if (state->offset_ram || state->offset_scratch) + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); /* Convert to KUNSEG and avoid RAM mirrors */ if (state->mirrors_mapped) { - imm = (s16)op->i.imm; + imm = (s16)c.i.imm; jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1)); - } else if (op->i.imm) { + } else if (c.i.imm) { imm = 0; - jit_addi(tmp, rs, (s16)op->i.imm); + jit_addi(tmp, rs, (s16)c.i.imm); jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1)); } else { imm = 0; @@ -873,7 +1216,7 @@ static void rec_store_direct_no_invalidate(const struct block *block, jit_movi(tmp2, state->offset_ram); - to_end = jit_jmpi(); + to_end = jit_b(); jit_patch(to_not_ram); jit_movi(tmp2, state->offset_scratch); @@ -882,60 +1225,74 @@ static void rec_store_direct_no_invalidate(const struct block *block, jit_movi(tmp2, state->offset_ram); } - if (state->offset_ram || state->offset_scratch) + if (state->offset_ram || state->offset_scratch) { jit_addr(tmp, tmp, tmp2); + lightrec_free_reg(reg_cache, tmp2); + } - lightrec_free_reg(reg_cache, tmp2); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + + if (is_big_endian() && swap_code && c.i.rt) { + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + + jit_new_node_ww(swap_code, tmp2, rt); + jit_new_node_www(code, imm, tmp, tmp2); - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rt); - jit_new_node_www(code, imm, tmp, rt); + lightrec_free_reg(reg_cache, tmp2); + } else { + jit_new_node_www(code, imm, tmp, rt); + } lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, tmp); } -static void rec_store_direct(const struct block *block, const struct opcode *op, - jit_code_t code) +static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block, + u16 offset, jit_code_t code, jit_code_t swap_code) { - struct lightrec_state *state = block->state; - struct regcache *reg_cache = state->reg_cache; + struct lightrec_state *state = cstate->state; + u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE; + struct regcache *reg_cache = cstate->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_end; u8 tmp, tmp2, tmp3, rs, rt; jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); - tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0); + tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); /* Convert to KUNSEG and avoid RAM mirrors */ - if (op->i.imm) { - jit_addi(tmp2, rs, (s16)op->i.imm); - jit_andi(tmp2, tmp2, 0x1f800000 | (RAM_SIZE - 1)); + if (c.i.imm) { + jit_addi(tmp2, rs, (s16)c.i.imm); + jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1)); } else { - jit_andi(tmp2, rs, 0x1f800000 | (RAM_SIZE - 1)); + jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1)); } lightrec_free_reg(reg_cache, rs); tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - to_not_ram = jit_bgti(tmp2, RAM_SIZE); + to_not_ram = jit_bgti(tmp2, ram_size); /* Compute the offset to the code LUT */ jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3); -#if __WORDSIZE == 64 - jit_lshi(tmp, tmp, 1); -#endif + if (!lut_is_32bit(state)) + jit_lshi(tmp, tmp, 1); jit_addr(tmp, LIGHTREC_REG_STATE, tmp); /* Write NULL to the code LUT to invalidate any block that's there */ - jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3); + if (lut_is_32bit(state)) + jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3); + else + jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3); if (state->offset_ram != state->offset_scratch) { jit_movi(tmp, state->offset_ram); - to_end = jit_jmpi(); + to_end = jit_b(); } jit_patch(to_not_ram); @@ -952,92 +1309,218 @@ static void rec_store_direct(const struct block *block, const struct opcode *op, lightrec_free_reg(reg_cache, tmp); lightrec_free_reg(reg_cache, tmp3); - rt = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rt); - jit_new_node_www(code, 0, tmp2, rt); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); + + if (is_big_endian() && swap_code && c.i.rt) { + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + + jit_new_node_ww(swap_code, tmp, rt); + jit_new_node_www(code, 0, tmp2, tmp); + + lightrec_free_reg(reg_cache, tmp); + } else { + jit_new_node_www(code, 0, tmp2, rt); + } lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, tmp2); } -static void rec_store(const struct block *block, const struct opcode *op, - jit_code_t code) -{ - if (op->flags & LIGHTREC_NO_INVALIDATE) { - rec_store_direct_no_invalidate(block, op, code); - } else if (op->flags & LIGHTREC_DIRECT_IO) { - if (block->state->invalidate_from_dma_only) - rec_store_direct_no_invalidate(block, op, code); - else - rec_store_direct(block, op, code); - } else { - rec_io(block, op, true, false); +static void rec_store(struct lightrec_cstate *state, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code) +{ + u16 flags = block->opcode_list[offset].flags; + bool no_invalidate = (flags & LIGHTREC_NO_INVALIDATE) || + state->state->invalidate_from_dma_only; + + switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { + case LIGHTREC_IO_RAM: + rec_store_ram(state, block, offset, code, + swap_code, !no_invalidate); + break; + case LIGHTREC_IO_SCRATCH: + rec_store_scratch(state, block, offset, code, swap_code); + break; + case LIGHTREC_IO_DIRECT: + if (no_invalidate) { + rec_store_direct_no_invalidate(state, block, offset, + code, swap_code); + } else { + rec_store_direct(state, block, offset, code, swap_code); + } + break; + default: + rec_io(state, block, offset, true, false); + break; } } -static void rec_SB(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SB(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_store(block, op, jit_code_stxi_c); + rec_store(state, block, offset, jit_code_stxi_c, 0); } -static void rec_SH(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SH(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_store(block, op, jit_code_stxi_s); + rec_store(state, block, offset, + jit_code_stxi_s, jit_code_bswapr_us); } -static void rec_SW(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SW(struct lightrec_cstate *state, + const struct block *block, u16 offset) + { _jit_name(block->_jit, __func__); - rec_store(block, op, jit_code_stxi_i); + rec_store(state, block, offset, + jit_code_stxi_i, jit_code_bswapr_ui); } -static void rec_SWL(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SWL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, true, false); + rec_io(state, block, offset, true, false); } -static void rec_SWR(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SWR(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, true, false); + rec_io(state, block, offset, true, false); } -static void rec_SWC2(const struct block *block, const struct opcode *op, u32 pc) +static void rec_SWC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, false, false); + rec_io(state, block, offset, false, false); } -static void rec_load_direct(const struct block *block, const struct opcode *op, - jit_code_t code) +static void rec_load_memory(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code, bool is_unsigned, + uintptr_t addr_offset, u32 addr_mask) { - struct lightrec_state *state = block->state; - struct regcache *reg_cache = state->reg_cache; + struct regcache *reg_cache = cstate->reg_cache; + struct opcode *op = &block->opcode_list[offset]; + jit_state_t *_jit = block->_jit; + u8 rs, rt, addr_reg, flags = REG_EXT; + bool no_mask = op->flags & LIGHTREC_NO_MASK; + union code c = op->c; + s16 imm; + + if (!c.i.rt) + return; + + if (is_unsigned) + flags |= REG_ZEXT; + + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); + + if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) { + jit_addi(rt, rs, (s16)c.i.imm); + addr_reg = rt; + imm = 0; + } else { + addr_reg = rs; + imm = (s16)c.i.imm; + } + + if (!no_mask) { + jit_andi(rt, addr_reg, addr_mask); + addr_reg = rt; + } + + if (addr_offset) { + jit_addi(rt, addr_reg, addr_offset); + addr_reg = rt; + } + + jit_new_node_www(code, rt, addr_reg, imm); + + if (is_big_endian() && swap_code) { + jit_new_node_ww(swap_code, rt, rt); + + if (c.i.op == OP_LH) + jit_extr_s(rt, rt); + else if (c.i.op == OP_LW && __WORDSIZE == 64) + jit_extr_i(rt, rt); + } + + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); +} + +static void rec_load_ram(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code, bool is_unsigned) +{ + _jit_note(block->_jit, __FILE__, __LINE__); + + rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, + cstate->state->offset_ram, rec_ram_mask(cstate->state)); +} + +static void rec_load_bios(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code, bool is_unsigned) +{ + _jit_note(block->_jit, __FILE__, __LINE__); + + rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, + cstate->state->offset_bios, 0x1fffffff); +} + +static void rec_load_scratch(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code, bool is_unsigned) +{ + _jit_note(block->_jit, __FILE__, __LINE__); + + rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, + cstate->state->offset_scratch, 0x1fffffff); +} + +static void rec_load_direct(struct lightrec_cstate *cstate, + const struct block *block, u16 offset, + jit_code_t code, jit_code_t swap_code, + bool is_unsigned) +{ + struct lightrec_state *state = cstate->state; + struct regcache *reg_cache = cstate->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2; - u8 tmp, rs, rt, addr_reg; + u8 tmp, rs, rt, addr_reg, flags = REG_EXT; s16 imm; - if (!op->i.rt) + if (!c.i.rt) return; + if (is_unsigned) + flags |= REG_ZEXT; + jit_note(__FILE__, __LINE__); - rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs); - rt = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->i.rt); + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); if ((state->offset_ram == state->offset_bios && state->offset_ram == state->offset_scratch && - state->mirrors_mapped) || !op->i.imm) { + state->mirrors_mapped) || !c.i.imm) { addr_reg = rs; - imm = (s16)op->i.imm; + imm = (s16)c.i.imm; } else { - jit_addi(rt, rs, (s16)op->i.imm); + jit_addi(rt, rs, (s16)c.i.imm); addr_reg = rt; imm = 0; - if (op->i.rs != op->i.rt) + if (c.i.rs != c.i.rt) lightrec_free_reg(reg_cache, rs); } @@ -1065,7 +1548,7 @@ static void rec_load_direct(const struct block *block, const struct opcode *op, if (state->offset_ram) jit_movi(tmp, state->offset_ram); - to_end = jit_jmpi(); + to_end = jit_b(); jit_patch(to_not_ram); @@ -1078,7 +1561,7 @@ static void rec_load_direct(const struct block *block, const struct opcode *op, jit_movi(tmp, state->offset_bios); if (state->offset_bios != state->offset_scratch) { - to_end2 = jit_jmpi(); + to_end2 = jit_b(); jit_patch(to_not_bios); @@ -1099,338 +1582,646 @@ static void rec_load_direct(const struct block *block, const struct opcode *op, jit_new_node_www(code, rt, rt, imm); + if (is_big_endian() && swap_code) { + jit_new_node_ww(swap_code, rt, rt); + + if (c.i.op == OP_LH) + jit_extr_s(rt, rt); + else if (c.i.op == OP_LW && __WORDSIZE == 64) + jit_extr_i(rt, rt); + } + lightrec_free_reg(reg_cache, addr_reg); lightrec_free_reg(reg_cache, rt); lightrec_free_reg(reg_cache, tmp); } -static void rec_load(const struct block *block, const struct opcode *op, - jit_code_t code) -{ - if (op->flags & LIGHTREC_DIRECT_IO) - rec_load_direct(block, op, code); - else - rec_io(block, op, false, true); +static void rec_load(struct lightrec_cstate *state, const struct block *block, + u16 offset, jit_code_t code, jit_code_t swap_code, + bool is_unsigned) +{ + u16 flags = block->opcode_list[offset].flags; + + switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { + case LIGHTREC_IO_RAM: + rec_load_ram(state, block, offset, code, swap_code, is_unsigned); + break; + case LIGHTREC_IO_BIOS: + rec_load_bios(state, block, offset, code, swap_code, is_unsigned); + break; + case LIGHTREC_IO_SCRATCH: + rec_load_scratch(state, block, offset, code, swap_code, is_unsigned); + break; + case LIGHTREC_IO_DIRECT: + rec_load_direct(state, block, offset, code, swap_code, is_unsigned); + break; + default: + rec_io(state, block, offset, false, true); + break; + } } -static void rec_LB(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_load(block, op, jit_code_ldxi_c); + rec_load(state, block, offset, jit_code_ldxi_c, 0, false); } -static void rec_LBU(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_load(block, op, jit_code_ldxi_uc); + rec_load(state, block, offset, jit_code_ldxi_uc, 0, true); } -static void rec_LH(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_load(block, op, jit_code_ldxi_s); + rec_load(state, block, offset, jit_code_ldxi_s, jit_code_bswapr_us, false); } -static void rec_LHU(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_load(block, op, jit_code_ldxi_us); + rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true); } -static void rec_LWL(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, true, true); + rec_io(state, block, offset, true, true); } -static void rec_LWR(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, true, true); + rec_io(state, block, offset, true, true); } -static void rec_LW(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_load(block, op, jit_code_ldxi_i); + rec_load(state, block, offset, jit_code_ldxi_i, jit_code_bswapr_ui, false); } -static void rec_LWC2(const struct block *block, const struct opcode *op, u32 pc) +static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_io(block, op, false, false); + rec_io(state, block, offset, false, false); } -static void rec_break_syscall(const struct block *block, - const struct opcode *op, u32 pc, bool is_break) +static void rec_break_syscall(struct lightrec_cstate *state, + const struct block *block, u16 offset, bool is_break) { - struct regcache *reg_cache = block->state->reg_cache; - jit_state_t *_jit = block->_jit; - u32 offset; - u8 tmp; - - jit_note(__FILE__, __LINE__); + _jit_note(block->_jit, __FILE__, __LINE__); if (is_break) - offset = offsetof(struct lightrec_state, break_func); + call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK); else - offset = offsetof(struct lightrec_state, syscall_func); - - tmp = lightrec_alloc_reg_temp(reg_cache, _jit); - jit_ldxi(tmp, LIGHTREC_REG_STATE, offset); - jit_callr(tmp); - lightrec_free_reg(reg_cache, tmp); - - lightrec_regcache_mark_live(reg_cache, _jit); + call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL); /* TODO: the return address should be "pc - 4" if we're a delay slot */ - lightrec_emit_end_of_block(block, op, pc, -1, pc, 31, 0, true); + lightrec_emit_end_of_block(state, block, offset, -1, + get_ds_pc(block, offset, 0), + 31, 0, true); } -static void rec_special_SYSCALL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_SYSCALL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_break_syscall(block, op, pc, false); + rec_break_syscall(state, block, offset, false); } -static void rec_special_BREAK(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_special_BREAK(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_break_syscall(block, op, pc, true); + rec_break_syscall(state, block, offset, true); } -static void rec_mfc(const struct block *block, const struct opcode *op) +static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset) { - u8 tmp, tmp2; - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; jit_note(__FILE__, __LINE__); + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); + lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); - tmp = lightrec_alloc_reg(reg_cache, _jit, JIT_R0); - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC); - jit_ldxi(tmp2, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, mfc_func)); + if (c.i.op == OP_CP0 && + !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) && + (c.r.rd == 12 || c.r.rd == 13)) + lightrec_emit_end_of_block(state, block, offset, -1, + get_ds_pc(block, offset, 1), + 0, 0, true); +} + +static void +rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset) +{ + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rt; - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true); + jit_note(__FILE__, __LINE__); - jit_movi(tmp, op->opcode); - jit_callr(tmp2); - lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT); - lightrec_regcache_mark_live(reg_cache, _jit); + jit_ldxi_i(rt, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.cp0[c.r.rd])); + + lightrec_free_reg(reg_cache, rt); +} + +static bool block_in_bios(const struct lightrec_cstate *state, + const struct block *block) +{ + const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS]; + u32 pc = kunseg(block->pc); + + return pc >= bios->pc && pc < bios->pc + bios->length; } -static void rec_mtc(const struct block *block, const struct opcode *op, u32 pc) +static void +rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) { - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; + const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 tmp, tmp2; + u8 rt, tmp = 0, tmp2, status; jit_note(__FILE__, __LINE__); - tmp = lightrec_alloc_reg(reg_cache, _jit, JIT_R0); - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); - jit_ldxi(tmp2, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, mtc_func)); + switch(c.r.rd) { + case 1: + case 4: + case 8: + case 14: + case 15: + /* Those registers are read-only */ + return; + default: + break; + } - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, false); - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, false); + if (block_in_bios(state, block) && c.r.rd == 12) { + /* If we are running code from the BIOS, handle writes to the + * Status register in C. BIOS code may toggle bit 16 which will + * map/unmap the RAM, while game code cannot do that. */ + rec_mtc(state, block, offset); + return; + } - jit_movi(tmp, op->opcode); - jit_callr(tmp2); - lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); - lightrec_regcache_mark_live(reg_cache, _jit); + if (c.r.rd != 13) { + jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]), + LIGHTREC_REG_STATE, rt); + } + + if (c.r.rd == 12 || c.r.rd == 13) { + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_ldxi_i(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.cp0[13])); + + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + } + + if (c.r.rd == 12) { + status = rt; + } else if (c.r.rd == 13) { + /* Cause = (Cause & ~0x0300) | (value & 0x0300) */ + jit_andi(tmp2, rt, 0x0300); + jit_ori(tmp, tmp, 0x0300); + jit_xori(tmp, tmp, 0x0300); + jit_orr(tmp, tmp, tmp2); + jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.cp0[12])); + jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]), + LIGHTREC_REG_STATE, tmp); + status = tmp2; + } + + if (c.r.rd == 12 || c.r.rd == 13) { + /* Exit dynarec in case there's a software interrupt. + * exit_flags = !!(status & tmp & 0x0300) & status; */ + jit_andr(tmp, tmp, status); + jit_andi(tmp, tmp, 0x0300); + jit_nei(tmp, tmp, 0); + jit_andr(tmp, tmp, status); + } + + if (c.r.rd == 12) { + /* Exit dynarec in case we unmask a hardware interrupt. + * exit_flags = !(~status & 0x401) */ + + jit_comr(tmp2, status); + jit_andi(tmp2, tmp2, 0x401); + jit_eqi(tmp2, tmp2, 0); + jit_orr(tmp, tmp, tmp2); + } + + if (c.r.rd == 12 || c.r.rd == 13) { + jit_stxi_i(offsetof(struct lightrec_state, exit_flags), + LIGHTREC_REG_STATE, tmp); + + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, tmp2); + } - if (op->i.op == OP_CP0 && (op->r.rd == 12 || op->r.rd == 13)) - lightrec_emit_end_of_block(block, op, pc, -1, pc + 4, 0, 0, true); + lightrec_free_reg(reg_cache, rt); + + if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) && + (c.r.rd == 12 || c.r.rd == 13)) + lightrec_emit_eob(state, block, offset + 1, true); } -static void rec_cp0_MFC0(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp0_MFC0(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_mfc(block, op); + rec_mfc0(state, block, offset); } -static void rec_cp0_CFC0(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp0_CFC0(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_mfc(block, op); + rec_mfc0(state, block, offset); } -static void rec_cp0_MTC0(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp0_MTC0(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_mtc(block, op, pc); + rec_mtc0(state, block, offset); } -static void rec_cp0_CTC0(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp0_CTC0(struct lightrec_cstate *state, + const struct block *block, u16 offset) { _jit_name(block->_jit, __func__); - rec_mtc(block, op, pc); + rec_mtc0(state, block, offset); } -static void rec_cp2_basic_MFC2(const struct block *block, - const struct opcode *op, u32 pc) +static unsigned int cp2d_i_offset(u8 reg) { - _jit_name(block->_jit, __func__); - rec_mfc(block, op); + return offsetof(struct lightrec_state, regs.cp2d[reg]); } -static void rec_cp2_basic_CFC2(const struct block *block, - const struct opcode *op, u32 pc) +static unsigned int cp2d_s_offset(u8 reg) { - _jit_name(block->_jit, __func__); - rec_mfc(block, op); + return cp2d_i_offset(reg) + is_big_endian() * 2; } -static void rec_cp2_basic_MTC2(const struct block *block, - const struct opcode *op, u32 pc) +static unsigned int cp2c_i_offset(u8 reg) { - _jit_name(block->_jit, __func__); - rec_mtc(block, op, pc); + return offsetof(struct lightrec_state, regs.cp2c[reg]); +} + +static unsigned int cp2c_s_offset(u8 reg) +{ + return cp2c_i_offset(reg) + is_big_endian() * 2; } -static void rec_cp2_basic_CTC2(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) { + struct regcache *reg_cache = state->reg_cache; + const union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + const u32 zext_regs = 0x300f0080; + u8 rt, tmp, tmp2, tmp3, out, flags; + u8 reg = c.r.rd == 15 ? 14 : c.r.rd; + unsigned int i; + + _jit_name(block->_jit, __func__); + + flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT; + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags); + + switch (reg) { + case 1: + case 3: + case 5: + case 8: + case 9: + case 10: + case 11: + jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); + break; + case 7: + case 16: + case 17: + case 18: + case 19: + jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); + break; + case 28: + case 29: + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); + + for (i = 0; i < 3; i++) { + out = i == 0 ? rt : tmp; + + jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i)); + jit_movi(tmp2, 0x1f); + jit_rshi(out, tmp, 7); + + jit_ltr(tmp3, tmp2, out); + jit_movnr(out, tmp2, tmp3); + + jit_gei(tmp2, out, 0); + jit_movzr(out, tmp2, tmp2); + + if (i > 0) { + jit_lshi(tmp, tmp, 5 * i); + jit_orr(rt, rt, tmp); + } + } + + + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, tmp2); + lightrec_free_reg(reg_cache, tmp3); + break; + default: + jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg)); + break; + } + + lightrec_free_reg(reg_cache, rt); +} + +static void rec_cp2_basic_CFC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) +{ + struct regcache *reg_cache = state->reg_cache; + const union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + u8 rt; + _jit_name(block->_jit, __func__); - rec_mtc(block, op, pc); + + switch (c.r.rd) { + case 4: + case 12: + case 20: + case 26: + case 27: + case 29: + case 30: + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT); + jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd)); + break; + default: + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT); + jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd)); + break; + } + + lightrec_free_reg(reg_cache, rt); } -static void rec_cp0_RFE(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct lightrec_state *state = block->state; + struct regcache *reg_cache = state->reg_cache; + const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 tmp; + jit_node_t *loop, *to_loop; + u8 rt, tmp, tmp2, flags = 0; - jit_name(__func__); - jit_note(__FILE__, __LINE__); + _jit_name(block->_jit, __func__); - tmp = lightrec_alloc_reg_temp(state->reg_cache, _jit); - jit_ldxi(tmp, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, rfe_func)); - jit_callr(tmp); - lightrec_free_reg(state->reg_cache, tmp); + if (c.r.rd == 31) + return; + + if (c.r.rd == 30) + flags |= REG_EXT; + + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags); + + switch (c.r.rd) { + case 15: + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13)); + + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14)); + + jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp); + jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2); + jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt); + + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, tmp2); + break; + case 28: + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + + jit_lshi(tmp, rt, 7); + jit_andi(tmp, tmp, 0xf80); + jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp); + + jit_lshi(tmp, rt, 2); + jit_andi(tmp, tmp, 0xf80); + jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp); + + jit_rshi(tmp, rt, 3); + jit_andi(tmp, tmp, 0xf80); + jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp); - lightrec_regcache_mark_live(state->reg_cache, _jit); + lightrec_free_reg(reg_cache, tmp); + break; + case 30: + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + + /* if (rt < 0) rt = ~rt; */ + jit_rshi(tmp, rt, 31); + jit_xorr(tmp, rt, tmp); + + /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */ + jit_lshi(tmp, tmp, 1); + jit_movi(tmp2, 33); + + /* Decrement tmp2 and right-shift the value by 1 until it equals zero */ + loop = jit_label(); + jit_subi(tmp2, tmp2, 1); + jit_rshi_u(tmp, tmp, 1); + to_loop = jit_bnei(tmp, 0); + + jit_patch_at(to_loop, loop); + + jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2); + jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt); + + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, tmp2); + break; + default: + jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt); + break; + } + + lightrec_free_reg(reg_cache, rt); } -static void rec_CP(const struct block *block, const struct opcode *op, u32 pc) +static void rec_cp2_basic_CTC2(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct regcache *reg_cache = block->state->reg_cache; + struct regcache *reg_cache = state->reg_cache; + const union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; - u8 tmp, tmp2; + u8 rt, tmp, tmp2; - jit_name(__func__); - jit_note(__FILE__, __LINE__); + _jit_name(block->_jit, __func__); - tmp = lightrec_alloc_reg(reg_cache, _jit, JIT_R0); - tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); + rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); - jit_ldxi(tmp2, LIGHTREC_REG_STATE, - offsetof(struct lightrec_state, cp_func)); + switch (c.r.rd) { + case 4: + case 12: + case 20: + case 26: + case 27: + case 29: + case 30: + jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt); + break; + case 31: + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); - jit_movi(tmp, op->opcode); - jit_callr(tmp2); - lightrec_free_reg(reg_cache, tmp); - lightrec_free_reg(reg_cache, tmp2); + jit_andi(tmp, rt, 0x7f87e000); + jit_nei(tmp, tmp, 0); + jit_lshi(tmp, tmp, 31); - lightrec_regcache_mark_live(reg_cache, _jit); + jit_andi(tmp2, rt, 0x7ffff000); + jit_orr(tmp, tmp2, tmp); + + jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp); + + lightrec_free_reg(reg_cache, tmp); + lightrec_free_reg(reg_cache, tmp2); + break; + + default: + jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt); + } + + lightrec_free_reg(reg_cache, rt); } -static void rec_meta_unload(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_cp0_RFE(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; jit_state_t *_jit = block->_jit; + u8 status, tmp; jit_name(__func__); jit_note(__FILE__, __LINE__); - pr_debug("Unloading reg %s\n", lightrec_reg_name(op->i.rs)); - lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true); -} + status = lightrec_alloc_reg_temp(reg_cache, _jit); + jit_ldxi_i(status, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.cp0[12])); -static void rec_meta_BEQZ(const struct block *block, - const struct opcode *op, u32 pc) -{ - _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_bnei, 0, false, true); + tmp = lightrec_alloc_reg_temp(reg_cache, _jit); + + /* status = ((status >> 2) & 0xf) | status & ~0xf; */ + jit_rshi(tmp, status, 2); + jit_andi(tmp, tmp, 0xf); + jit_andi(status, status, ~0xful); + jit_orr(status, status, tmp); + + jit_ldxi_i(tmp, LIGHTREC_REG_STATE, + offsetof(struct lightrec_state, regs.cp0[13])); + jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]), + LIGHTREC_REG_STATE, status); + + /* Exit dynarec in case there's a software interrupt. + * exit_flags = !!(status & cause & 0x0300) & status; */ + jit_andr(tmp, tmp, status); + jit_andi(tmp, tmp, 0x0300); + jit_nei(tmp, tmp, 0); + jit_andr(tmp, tmp, status); + jit_stxi_i(offsetof(struct lightrec_state, exit_flags), + LIGHTREC_REG_STATE, tmp); + + lightrec_free_reg(reg_cache, status); + lightrec_free_reg(reg_cache, tmp); } -static void rec_meta_BNEZ(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_CP(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - _jit_name(block->_jit, __func__); - rec_b(block, op, pc, jit_code_beqi, 0, false, true); + union code c = block->opcode_list[offset].c; + jit_state_t *_jit = block->_jit; + + jit_name(__func__); + jit_note(__FILE__, __LINE__); + + call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP); } -static void rec_meta_MOV(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_meta_MOV(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - struct lightrec_state *state = block->state; struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; u8 rs, rd; _jit_name(block->_jit, __func__); jit_note(__FILE__, __LINE__); - rs = op->r.rs ? lightrec_alloc_reg_in(reg_cache, _jit, op->r.rs) : 0; - rd = lightrec_alloc_reg_out_ext(reg_cache, _jit, op->r.rd); + if (c.r.rs) + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); + rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT); - if (op->r.rs == 0) { + if (c.r.rs == 0) jit_movi(rd, 0); - } else { -#if __WORDSIZE == 32 - jit_movr(rd, rs); -#else + else jit_extr_i(rd, rs); -#endif - } - lightrec_free_reg(state->reg_cache, rs); - lightrec_free_reg(state->reg_cache, rd); + if (c.r.rs) + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rd); } -static void rec_meta_sync(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state, + const struct block *block, + u16 offset) { - struct lightrec_state *state = block->state; - struct lightrec_branch_target *target; + struct regcache *reg_cache = state->reg_cache; + union code c = block->opcode_list[offset].c; jit_state_t *_jit = block->_jit; + u8 rs, rt; - jit_name(__func__); + _jit_name(block->_jit, __func__); jit_note(__FILE__, __LINE__); - jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); - state->cycles = 0; + rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); + rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT); - lightrec_storeback_regs(state->reg_cache, _jit); - lightrec_regcache_reset(state->reg_cache); + if (c.i.op == OP_META_EXTC) + jit_extr_c(rt, rs); + else + jit_extr_s(rt, rs); - pr_debug("Adding branch target at offset 0x%x\n", - op->offset << 2); - target = &state->targets[state->nb_targets++]; - target->offset = op->offset; - target->label = jit_label(); + lightrec_free_reg(reg_cache, rs); + lightrec_free_reg(reg_cache, rt); } static const lightrec_rec_func_t rec_standard[64] = { + SET_DEFAULT_ELM(rec_standard, unknown_opcode), [OP_SPECIAL] = rec_SPECIAL, [OP_REGIMM] = rec_REGIMM, [OP_J] = rec_J, @@ -1464,14 +2255,13 @@ static const lightrec_rec_func_t rec_standard[64] = { [OP_LWC2] = rec_LWC2, [OP_SWC2] = rec_SWC2, - [OP_META_REG_UNLOAD] = rec_meta_unload, - [OP_META_BEQZ] = rec_meta_BEQZ, - [OP_META_BNEZ] = rec_meta_BNEZ, [OP_META_MOV] = rec_meta_MOV, - [OP_META_SYNC] = rec_meta_sync, + [OP_META_EXTC] = rec_meta_EXTC_EXTS, + [OP_META_EXTS] = rec_meta_EXTC_EXTS, }; static const lightrec_rec_func_t rec_special[64] = { + SET_DEFAULT_ELM(rec_special, unknown_opcode), [OP_SPECIAL_SLL] = rec_special_SLL, [OP_SPECIAL_SRL] = rec_special_SRL, [OP_SPECIAL_SRA] = rec_special_SRA, @@ -1503,6 +2293,7 @@ static const lightrec_rec_func_t rec_special[64] = { }; static const lightrec_rec_func_t rec_regimm[64] = { + SET_DEFAULT_ELM(rec_regimm, unknown_opcode), [OP_REGIMM_BLTZ] = rec_regimm_BLTZ, [OP_REGIMM_BGEZ] = rec_regimm_BGEZ, [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL, @@ -1510,6 +2301,7 @@ static const lightrec_rec_func_t rec_regimm[64] = { }; static const lightrec_rec_func_t rec_cp0[64] = { + SET_DEFAULT_ELM(rec_cp0, rec_CP), [OP_CP0_MFC0] = rec_cp0_MFC0, [OP_CP0_CFC0] = rec_cp0_CFC0, [OP_CP0_MTC0] = rec_cp0_MTC0, @@ -1518,60 +2310,107 @@ static const lightrec_rec_func_t rec_cp0[64] = { }; static const lightrec_rec_func_t rec_cp2_basic[64] = { + SET_DEFAULT_ELM(rec_cp2_basic, rec_CP), [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2, [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2, [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2, [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2, }; -static void rec_SPECIAL(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_SPECIAL(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - lightrec_rec_func_t f = rec_special[op->r.op]; - if (likely(f)) - (*f)(block, op, pc); + union code c = block->opcode_list[offset].c; + lightrec_rec_func_t f = rec_special[c.r.op]; + + if (!HAS_DEFAULT_ELM && unlikely(!f)) + unknown_opcode(state, block, offset); else - unknown_opcode(block, op, pc); + (*f)(state, block, offset); } -static void rec_REGIMM(const struct block *block, - const struct opcode *op, u32 pc) +static void rec_REGIMM(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - lightrec_rec_func_t f = rec_regimm[op->r.rt]; - if (likely(f)) - (*f)(block, op, pc); + union code c = block->opcode_list[offset].c; + lightrec_rec_func_t f = rec_regimm[c.r.rt]; + + if (!HAS_DEFAULT_ELM && unlikely(!f)) + unknown_opcode(state, block, offset); else - unknown_opcode(block, op, pc); + (*f)(state, block, offset); } -static void rec_CP0(const struct block *block, const struct opcode *op, u32 pc) +static void rec_CP0(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - lightrec_rec_func_t f = rec_cp0[op->r.rs]; - if (likely(f)) - (*f)(block, op, pc); + union code c = block->opcode_list[offset].c; + lightrec_rec_func_t f = rec_cp0[c.r.rs]; + + if (!HAS_DEFAULT_ELM && unlikely(!f)) + rec_CP(state, block, offset); else - rec_CP(block, op, pc); + (*f)(state, block, offset); } -static void rec_CP2(const struct block *block, const struct opcode *op, u32 pc) +static void rec_CP2(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - if (op->r.op == OP_CP2_BASIC) { - lightrec_rec_func_t f = rec_cp2_basic[op->r.rs]; - if (likely(f)) { - (*f)(block, op, pc); + union code c = block->opcode_list[offset].c; + + if (c.r.op == OP_CP2_BASIC) { + lightrec_rec_func_t f = rec_cp2_basic[c.r.rs]; + + if (HAS_DEFAULT_ELM || likely(f)) { + (*f)(state, block, offset); return; } } - rec_CP(block, op, pc); + rec_CP(state, block, offset); } -void lightrec_rec_opcode(const struct block *block, - const struct opcode *op, u32 pc) +void lightrec_rec_opcode(struct lightrec_cstate *state, + const struct block *block, u16 offset) { - lightrec_rec_func_t f = rec_standard[op->i.op]; - if (likely(f)) - (*f)(block, op, pc); - else - unknown_opcode(block, op, pc); + struct regcache *reg_cache = state->reg_cache; + struct lightrec_branch_target *target; + const struct opcode *op = &block->opcode_list[offset]; + jit_state_t *_jit = block->_jit; + lightrec_rec_func_t f; + + if (op->flags & LIGHTREC_SYNC) { + jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); + state->cycles = 0; + + lightrec_storeback_regs(reg_cache, _jit); + lightrec_regcache_reset(reg_cache); + + pr_debug("Adding branch target at offset 0x%x\n", offset << 2); + target = &state->targets[state->nb_targets++]; + target->offset = offset; + target->label = jit_indirect(); + } + + if (likely(op->opcode)) { + f = rec_standard[op->i.op]; + + if (!HAS_DEFAULT_ELM && unlikely(!f)) + unknown_opcode(state, block, offset); + else + (*f)(state, block, offset); + } + + if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) { + lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true); + pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd)); + } + if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) { + lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true); + pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt)); + } + if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) { + lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true); + pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt)); + } }