1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "disassembler.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
24 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
25 const struct block *block, u16 offset, u8 reg, u8 in_reg);
26 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
27 const struct block *block, u16 offset,
30 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
32 pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
33 block->opcode_list[offset].c.opcode,
34 block->pc + (offset << 2));
38 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
40 /* Prevent jit_jmpi() from using our cycles register as a temporary */
41 jit_live(LIGHTREC_REG_CYCLE);
43 jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
46 static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
47 u8 ra_reg, u32 pc, u32 link)
51 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
52 lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
53 lightrec_free_reg(reg_cache, link_reg);
56 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
57 const struct block *block, u16 offset,
58 s8 reg_new_pc, u32 imm, u8 ra_reg,
59 u32 link, bool update_cycles)
61 struct regcache *reg_cache = state->reg_cache;
62 jit_state_t *_jit = block->_jit;
63 const struct opcode *op = &block->opcode_list[offset],
64 *next = &block->opcode_list[offset + 1];
65 u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
67 jit_note(__FILE__, __LINE__);
69 if (link && ra_reg != reg_new_pc)
70 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
73 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
75 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
77 if (link && ra_reg == reg_new_pc) {
78 /* Handle the special case: JALR $r0, $r0
79 * In that case the target PC should be the old value of the
81 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
84 if (has_delay_slot(op->c) &&
85 !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
86 cycles += lightrec_cycles_of_opcode(next->c);
88 /* Recompile the delay slot */
90 lightrec_rec_opcode(state, block, offset + 1);
93 /* Clean the remaining registers */
94 lightrec_clean_regs(reg_cache, _jit);
96 if (cycles && update_cycles) {
97 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
98 pr_debug("EOB: %u cycles\n", cycles);
101 lightrec_jump_to_eob(state, _jit);
104 void lightrec_emit_eob(struct lightrec_cstate *state,
105 const struct block *block, u16 offset)
107 struct regcache *reg_cache = state->reg_cache;
108 jit_state_t *_jit = block->_jit;
110 lightrec_clean_regs(reg_cache, _jit);
112 lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
113 block->pc + (offset << 2));
114 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
116 lightrec_jump_to_eob(state, _jit);
119 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
121 union code c = block->opcode_list[offset].c;
123 _jit_name(block->_jit, __func__);
124 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
127 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
129 union code c = block->opcode_list[offset].c;
131 _jit_name(block->_jit, __func__);
132 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
133 get_branch_pc(block, offset, 2), true);
136 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
138 union code c = block->opcode_list[offset].c;
140 _jit_name(block->_jit, __func__);
141 lightrec_emit_end_of_block(state, block, offset, -1,
142 (block->pc & 0xf0000000) | (c.j.imm << 2),
146 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
148 union code c = block->opcode_list[offset].c;
150 _jit_name(block->_jit, __func__);
151 lightrec_emit_end_of_block(state, block, offset, -1,
152 (block->pc & 0xf0000000) | (c.j.imm << 2),
153 31, get_branch_pc(block, offset, 2), true);
156 static void lightrec_do_early_unload(struct lightrec_cstate *state,
157 const struct block *block, u16 offset)
159 struct regcache *reg_cache = state->reg_cache;
160 const struct opcode *op = &block->opcode_list[offset];
161 jit_state_t *_jit = block->_jit;
167 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
168 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
169 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
172 for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
173 reg = reg_ops[i].reg;
175 switch (reg_ops[i].op) {
176 case LIGHTREC_REG_UNLOAD:
177 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
180 case LIGHTREC_REG_DISCARD:
181 lightrec_discard_reg_if_loaded(reg_cache, reg);
184 case LIGHTREC_REG_CLEAN:
185 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
193 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
194 jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
196 struct regcache *reg_cache = state->reg_cache;
197 struct native_register *regs_backup;
198 jit_state_t *_jit = block->_jit;
199 struct lightrec_branch *branch;
200 const struct opcode *op = &block->opcode_list[offset],
201 *next = &block->opcode_list[offset + 1];
203 bool is_forward = (s16)op->i.imm >= -1;
204 int op_cycles = lightrec_cycles_of_opcode(op->c);
205 u32 target_offset, cycles = state->cycles + op_cycles;
206 bool no_indirection = false;
210 jit_note(__FILE__, __LINE__);
212 if (!op_flag_no_ds(op->flags))
213 cycles += lightrec_cycles_of_opcode(next->c);
215 state->cycles = -op_cycles;
217 if (!unconditional) {
218 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
219 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
220 _jit, op->i.rt, REG_EXT);
222 /* Unload dead registers before evaluating the branch */
223 if (OPT_EARLY_UNLOAD)
224 lightrec_do_early_unload(state, block, offset);
226 if (op_flag_local_branch(op->flags) &&
227 (op_flag_no_ds(op->flags) || !next->opcode) &&
228 is_forward && !lightrec_has_dirty_regs(reg_cache))
229 no_indirection = true;
232 pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
236 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
238 if (!unconditional) {
239 /* Generate the branch opcode */
241 addr = jit_new_node_pww(code, NULL, rs, rt);
243 lightrec_free_regs(reg_cache);
244 regs_backup = lightrec_regcache_enter_branch(reg_cache);
247 if (op_flag_local_branch(op->flags)) {
248 /* Recompile the delay slot */
249 if (!op_flag_no_ds(op->flags) && next->opcode)
250 lightrec_rec_opcode(state, block, offset + 1);
253 update_ra_register(reg_cache, _jit, 31, block->pc, link);
255 /* Clean remaining registers */
256 lightrec_clean_regs(reg_cache, _jit);
258 target_offset = offset + 1 + (s16)op->i.imm
259 - !!op_flag_no_ds(op->flags);
260 pr_debug("Adding local branch to offset 0x%x\n",
262 branch = &state->local_branches[
263 state->nb_local_branches++];
265 branch->target = target_offset;
268 branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
270 branch->branch = jit_b();
272 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
275 if (!op_flag_local_branch(op->flags) || !is_forward) {
276 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
277 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
281 if (!unconditional) {
285 lightrec_regcache_leave_branch(reg_cache, regs_backup);
288 update_ra_register(reg_cache, _jit, 31, block->pc, link);
290 if (!op_flag_no_ds(op->flags) && next->opcode)
291 lightrec_rec_opcode(state, block, offset + 1);
295 static void rec_BNE(struct lightrec_cstate *state,
296 const struct block *block, u16 offset)
298 union code c = block->opcode_list[offset].c;
300 _jit_name(block->_jit, __func__);
303 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
305 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
308 static void rec_BEQ(struct lightrec_cstate *state,
309 const struct block *block, u16 offset)
311 union code c = block->opcode_list[offset].c;
313 _jit_name(block->_jit, __func__);
316 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
318 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
321 static void rec_BLEZ(struct lightrec_cstate *state,
322 const struct block *block, u16 offset)
324 union code c = block->opcode_list[offset].c;
326 _jit_name(block->_jit, __func__);
327 rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
330 static void rec_BGTZ(struct lightrec_cstate *state,
331 const struct block *block, u16 offset)
333 _jit_name(block->_jit, __func__);
334 rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
337 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
338 const struct block *block, u16 offset)
340 _jit_name(block->_jit, __func__);
341 rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
344 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
345 const struct block *block, u16 offset)
347 _jit_name(block->_jit, __func__);
348 rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
349 get_branch_pc(block, offset, 2), false, true);
352 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
353 const struct block *block, u16 offset)
355 union code c = block->opcode_list[offset].c;
357 _jit_name(block->_jit, __func__);
358 rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
361 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
362 const struct block *block, u16 offset)
364 const struct opcode *op = &block->opcode_list[offset];
365 _jit_name(block->_jit, __func__);
366 rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
367 get_branch_pc(block, offset, 2),
371 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
372 u16 offset, jit_code_t code, bool slti)
374 struct regcache *reg_cache = state->reg_cache;
375 union code c = block->opcode_list[offset].c;
376 jit_state_t *_jit = block->_jit;
377 u8 rs, rt, out_flags = REG_EXT;
380 out_flags |= REG_ZEXT;
382 jit_note(__FILE__, __LINE__);
383 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
384 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
386 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
388 lightrec_free_reg(reg_cache, rs);
389 lightrec_free_reg(reg_cache, rt);
392 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
393 u16 offset, jit_code_t code, bool out_ext)
395 struct regcache *reg_cache = state->reg_cache;
396 union code c = block->opcode_list[offset].c;
397 jit_state_t *_jit = block->_jit;
400 jit_note(__FILE__, __LINE__);
401 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
402 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
403 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
404 out_ext ? REG_EXT | REG_ZEXT : 0);
406 jit_new_node_www(code, rd, rs, rt);
408 lightrec_free_reg(reg_cache, rs);
409 lightrec_free_reg(reg_cache, rt);
410 lightrec_free_reg(reg_cache, rd);
413 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
414 u16 offset, jit_code_t code)
416 struct regcache *reg_cache = state->reg_cache;
417 union code c = block->opcode_list[offset].c;
418 jit_state_t *_jit = block->_jit;
419 u8 rd, rt, rs, temp, flags = 0;
421 jit_note(__FILE__, __LINE__);
422 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
424 if (code == jit_code_rshr)
426 else if (code == jit_code_rshr_u)
429 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
430 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
432 if (rs != rd && rt != rd) {
433 jit_andi(rd, rs, 0x1f);
434 jit_new_node_www(code, rd, rt, rd);
436 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
437 jit_andi(temp, rs, 0x1f);
438 jit_new_node_www(code, rd, rt, temp);
439 lightrec_free_reg(reg_cache, temp);
442 lightrec_free_reg(reg_cache, rs);
443 lightrec_free_reg(reg_cache, rt);
444 lightrec_free_reg(reg_cache, rd);
447 static void rec_movi(struct lightrec_cstate *state,
448 const struct block *block, u16 offset)
450 struct regcache *reg_cache = state->reg_cache;
451 union code c = block->opcode_list[offset].c;
452 jit_state_t *_jit = block->_jit;
456 if (!(c.i.imm & 0x8000))
459 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
461 jit_movi(rt, (s32)(s16) c.i.imm);
463 lightrec_free_reg(reg_cache, rt);
466 static void rec_ADDIU(struct lightrec_cstate *state,
467 const struct block *block, u16 offset)
469 _jit_name(block->_jit, __func__);
471 if (block->opcode_list[offset].c.i.rs)
472 rec_alu_imm(state, block, offset, jit_code_addi, false);
474 rec_movi(state, block, offset);
477 static void rec_ADDI(struct lightrec_cstate *state,
478 const struct block *block, u16 offset)
480 /* TODO: Handle the exception? */
481 _jit_name(block->_jit, __func__);
482 rec_ADDIU(state, block, offset);
485 static void rec_SLTIU(struct lightrec_cstate *state,
486 const struct block *block, u16 offset)
488 _jit_name(block->_jit, __func__);
489 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
492 static void rec_SLTI(struct lightrec_cstate *state,
493 const struct block *block, u16 offset)
495 _jit_name(block->_jit, __func__);
496 rec_alu_imm(state, block, offset, jit_code_lti, true);
499 static void rec_ANDI(struct lightrec_cstate *state,
500 const struct block *block, u16 offset)
502 struct regcache *reg_cache = state->reg_cache;
503 union code c = block->opcode_list[offset].c;
504 jit_state_t *_jit = block->_jit;
507 _jit_name(block->_jit, __func__);
508 jit_note(__FILE__, __LINE__);
509 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
510 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
513 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
514 * casts to uint8_t / uint16_t. */
517 else if (c.i.imm == 0xffff)
520 jit_andi(rt, rs, (u32)(u16) c.i.imm);
522 lightrec_free_reg(reg_cache, rs);
523 lightrec_free_reg(reg_cache, rt);
526 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
527 u16 offset, jit_code_t code)
529 struct regcache *reg_cache = state->reg_cache;
530 union code c = block->opcode_list[offset].c;
531 jit_state_t *_jit = block->_jit;
534 jit_note(__FILE__, __LINE__);
535 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
536 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
538 flags = lightrec_get_reg_in_flags(reg_cache, rs);
539 lightrec_set_reg_out_flags(reg_cache, rt, flags);
541 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
543 lightrec_free_reg(reg_cache, rs);
544 lightrec_free_reg(reg_cache, rt);
548 static void rec_ORI(struct lightrec_cstate *state,
549 const struct block *block, u16 offset)
551 _jit_name(block->_jit, __func__);
552 rec_alu_or_xor(state, block, offset, jit_code_ori);
555 static void rec_XORI(struct lightrec_cstate *state,
556 const struct block *block, u16 offset)
558 _jit_name(block->_jit, __func__);
559 rec_alu_or_xor(state, block, offset, jit_code_xori);
562 static void rec_LUI(struct lightrec_cstate *state,
563 const struct block *block, u16 offset)
565 struct regcache *reg_cache = state->reg_cache;
566 union code c = block->opcode_list[offset].c;
567 jit_state_t *_jit = block->_jit;
568 u8 rt, flags = REG_EXT;
571 jit_note(__FILE__, __LINE__);
573 if (!(c.i.imm & BIT(15)))
576 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
578 jit_movi(rt, (s32)(c.i.imm << 16));
580 lightrec_free_reg(reg_cache, rt);
583 static void rec_special_ADDU(struct lightrec_cstate *state,
584 const struct block *block, u16 offset)
586 _jit_name(block->_jit, __func__);
587 rec_alu_special(state, block, offset, jit_code_addr, false);
590 static void rec_special_ADD(struct lightrec_cstate *state,
591 const struct block *block, u16 offset)
593 /* TODO: Handle the exception? */
594 _jit_name(block->_jit, __func__);
595 rec_alu_special(state, block, offset, jit_code_addr, false);
598 static void rec_special_SUBU(struct lightrec_cstate *state,
599 const struct block *block, u16 offset)
601 _jit_name(block->_jit, __func__);
602 rec_alu_special(state, block, offset, jit_code_subr, false);
605 static void rec_special_SUB(struct lightrec_cstate *state,
606 const struct block *block, u16 offset)
608 /* TODO: Handle the exception? */
609 _jit_name(block->_jit, __func__);
610 rec_alu_special(state, block, offset, jit_code_subr, false);
613 static void rec_special_AND(struct lightrec_cstate *state,
614 const struct block *block, u16 offset)
616 struct regcache *reg_cache = state->reg_cache;
617 union code c = block->opcode_list[offset].c;
618 jit_state_t *_jit = block->_jit;
619 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
621 _jit_name(block->_jit, __func__);
622 jit_note(__FILE__, __LINE__);
623 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
624 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
625 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
627 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
628 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
630 /* Z(rd) = Z(rs) | Z(rt) */
631 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
633 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
634 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
635 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
636 (REG_EXT & flags_rs & flags_rt))
639 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
641 jit_andr(rd, rs, rt);
643 lightrec_free_reg(reg_cache, rs);
644 lightrec_free_reg(reg_cache, rt);
645 lightrec_free_reg(reg_cache, rd);
648 static void rec_special_or_nor(struct lightrec_cstate *state,
649 const struct block *block, u16 offset, bool nor)
651 struct regcache *reg_cache = state->reg_cache;
652 union code c = block->opcode_list[offset].c;
653 jit_state_t *_jit = block->_jit;
654 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
656 jit_note(__FILE__, __LINE__);
657 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
658 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
659 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
661 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
662 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
664 /* or: Z(rd) = Z(rs) & Z(rt)
667 flags_rd = REG_ZEXT & flags_rs & flags_rt;
669 /* E(rd) = E(rs) & E(rt) */
670 if (REG_EXT & flags_rs & flags_rt)
673 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
680 lightrec_free_reg(reg_cache, rs);
681 lightrec_free_reg(reg_cache, rt);
682 lightrec_free_reg(reg_cache, rd);
685 static void rec_special_OR(struct lightrec_cstate *state,
686 const struct block *block, u16 offset)
688 _jit_name(block->_jit, __func__);
689 rec_special_or_nor(state, block, offset, false);
692 static void rec_special_NOR(struct lightrec_cstate *state,
693 const struct block *block, u16 offset)
695 _jit_name(block->_jit, __func__);
696 rec_special_or_nor(state, block, offset, true);
699 static void rec_special_XOR(struct lightrec_cstate *state,
700 const struct block *block, u16 offset)
702 struct regcache *reg_cache = state->reg_cache;
703 union code c = block->opcode_list[offset].c;
704 jit_state_t *_jit = block->_jit;
705 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
707 _jit_name(block->_jit, __func__);
709 jit_note(__FILE__, __LINE__);
710 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
711 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
712 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
714 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
715 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
717 /* Z(rd) = Z(rs) & Z(rt) */
718 flags_rd = REG_ZEXT & flags_rs & flags_rt;
720 /* E(rd) = E(rs) & E(rt) */
721 flags_rd |= REG_EXT & flags_rs & flags_rt;
723 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
725 jit_xorr(rd, rs, rt);
727 lightrec_free_reg(reg_cache, rs);
728 lightrec_free_reg(reg_cache, rt);
729 lightrec_free_reg(reg_cache, rd);
732 static void rec_special_SLTU(struct lightrec_cstate *state,
733 const struct block *block, u16 offset)
735 _jit_name(block->_jit, __func__);
736 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
739 static void rec_special_SLT(struct lightrec_cstate *state,
740 const struct block *block, u16 offset)
742 _jit_name(block->_jit, __func__);
743 rec_alu_special(state, block, offset, jit_code_ltr, true);
746 static void rec_special_SLLV(struct lightrec_cstate *state,
747 const struct block *block, u16 offset)
749 _jit_name(block->_jit, __func__);
750 rec_alu_shiftv(state, block, offset, jit_code_lshr);
753 static void rec_special_SRLV(struct lightrec_cstate *state,
754 const struct block *block, u16 offset)
756 _jit_name(block->_jit, __func__);
757 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
760 static void rec_special_SRAV(struct lightrec_cstate *state,
761 const struct block *block, u16 offset)
763 _jit_name(block->_jit, __func__);
764 rec_alu_shiftv(state, block, offset, jit_code_rshr);
767 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
768 u16 offset, jit_code_t code)
770 struct regcache *reg_cache = state->reg_cache;
771 union code c = block->opcode_list[offset].c;
772 jit_state_t *_jit = block->_jit;
773 u8 rd, rt, flags = 0;
775 jit_note(__FILE__, __LINE__);
777 if (code == jit_code_rshi)
779 else if (code == jit_code_rshi_u)
782 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
784 /* Input reg is zero-extended, if we SRL at least by one bit, we know
785 * the output reg will be both zero-extended and sign-extended. */
786 if (code == jit_code_rshi_u && c.r.imm)
788 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
790 jit_new_node_www(code, rd, rt, c.r.imm);
792 lightrec_free_reg(reg_cache, rt);
793 lightrec_free_reg(reg_cache, rd);
796 static void rec_special_SLL(struct lightrec_cstate *state,
797 const struct block *block, u16 offset)
799 _jit_name(block->_jit, __func__);
800 rec_alu_shift(state, block, offset, jit_code_lshi);
803 static void rec_special_SRL(struct lightrec_cstate *state,
804 const struct block *block, u16 offset)
806 _jit_name(block->_jit, __func__);
807 rec_alu_shift(state, block, offset, jit_code_rshi_u);
810 static void rec_special_SRA(struct lightrec_cstate *state,
811 const struct block *block, u16 offset)
813 _jit_name(block->_jit, __func__);
814 rec_alu_shift(state, block, offset, jit_code_rshi);
817 static void rec_alu_mult(struct lightrec_cstate *state,
818 const struct block *block, u16 offset, bool is_signed)
820 struct regcache *reg_cache = state->reg_cache;
821 union code c = block->opcode_list[offset].c;
822 u32 flags = block->opcode_list[offset].flags;
823 u8 reg_lo = get_mult_div_lo(c);
824 u8 reg_hi = get_mult_div_hi(c);
825 jit_state_t *_jit = block->_jit;
826 u8 lo, hi, rs, rt, rflags = 0;
828 jit_note(__FILE__, __LINE__);
835 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
836 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
838 if (!op_flag_no_lo(flags))
839 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
840 else if (__WORDSIZE == 32)
841 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
843 if (!op_flag_no_hi(flags))
844 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
846 if (__WORDSIZE == 32) {
847 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
848 * operation if the MULT was detected a 32-bit only. */
849 if (!op_flag_no_hi(flags)) {
851 jit_qmulr(lo, hi, rs, rt);
853 jit_qmulr_u(lo, hi, rs, rt);
855 jit_mulr(lo, rs, rt);
858 /* On 64-bit systems, do a 64*64->64 bit operation. */
859 if (op_flag_no_lo(flags)) {
860 jit_mulr(hi, rs, rt);
861 jit_rshi(hi, hi, 32);
863 jit_mulr(lo, rs, rt);
865 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
866 if (!op_flag_no_hi(flags))
867 jit_rshi(hi, lo, 32);
871 lightrec_free_reg(reg_cache, rs);
872 lightrec_free_reg(reg_cache, rt);
873 if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
874 lightrec_free_reg(reg_cache, lo);
875 if (!op_flag_no_hi(flags))
876 lightrec_free_reg(reg_cache, hi);
879 static void rec_alu_div(struct lightrec_cstate *state,
880 const struct block *block, u16 offset, bool is_signed)
882 struct regcache *reg_cache = state->reg_cache;
883 union code c = block->opcode_list[offset].c;
884 u32 flags = block->opcode_list[offset].flags;
885 bool no_check = op_flag_no_div_check(flags);
886 u8 reg_lo = get_mult_div_lo(c);
887 u8 reg_hi = get_mult_div_hi(c);
888 jit_state_t *_jit = block->_jit;
889 jit_node_t *branch, *to_end;
890 u8 lo = 0, hi = 0, rs, rt, rflags = 0;
892 jit_note(__FILE__, __LINE__);
899 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
900 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
902 if (!op_flag_no_lo(flags))
903 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
905 if (!op_flag_no_hi(flags))
906 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
908 /* Jump to special handler if dividing by zero */
910 branch = jit_beqi(rt, 0);
912 if (op_flag_no_lo(flags)) {
914 jit_remr(hi, rs, rt);
916 jit_remr_u(hi, rs, rt);
917 } else if (op_flag_no_hi(flags)) {
919 jit_divr(lo, rs, rt);
921 jit_divr_u(lo, rs, rt);
924 jit_qdivr(lo, hi, rs, rt);
926 jit_qdivr_u(lo, hi, rs, rt);
930 /* Jump above the div-by-zero handler */
935 if (!op_flag_no_lo(flags)) {
945 if (!op_flag_no_hi(flags))
951 lightrec_free_reg(reg_cache, rs);
952 lightrec_free_reg(reg_cache, rt);
954 if (!op_flag_no_lo(flags))
955 lightrec_free_reg(reg_cache, lo);
957 if (!op_flag_no_hi(flags))
958 lightrec_free_reg(reg_cache, hi);
961 static void rec_special_MULT(struct lightrec_cstate *state,
962 const struct block *block, u16 offset)
964 _jit_name(block->_jit, __func__);
965 rec_alu_mult(state, block, offset, true);
968 static void rec_special_MULTU(struct lightrec_cstate *state,
969 const struct block *block, u16 offset)
971 _jit_name(block->_jit, __func__);
972 rec_alu_mult(state, block, offset, false);
975 static void rec_special_DIV(struct lightrec_cstate *state,
976 const struct block *block, u16 offset)
978 _jit_name(block->_jit, __func__);
979 rec_alu_div(state, block, offset, true);
982 static void rec_special_DIVU(struct lightrec_cstate *state,
983 const struct block *block, u16 offset)
985 _jit_name(block->_jit, __func__);
986 rec_alu_div(state, block, offset, false);
989 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
990 const struct block *block, u8 dst, u8 src)
992 struct regcache *reg_cache = state->reg_cache;
993 jit_state_t *_jit = block->_jit;
995 jit_note(__FILE__, __LINE__);
996 src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
997 dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
999 jit_extr_i(dst, src);
1001 lightrec_free_reg(reg_cache, src);
1002 lightrec_free_reg(reg_cache, dst);
1005 static void rec_special_MFHI(struct lightrec_cstate *state,
1006 const struct block *block, u16 offset)
1008 union code c = block->opcode_list[offset].c;
1010 _jit_name(block->_jit, __func__);
1011 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1014 static void rec_special_MTHI(struct lightrec_cstate *state,
1015 const struct block *block, u16 offset)
1017 union code c = block->opcode_list[offset].c;
1019 _jit_name(block->_jit, __func__);
1020 rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1023 static void rec_special_MFLO(struct lightrec_cstate *state,
1024 const struct block *block, u16 offset)
1026 union code c = block->opcode_list[offset].c;
1028 _jit_name(block->_jit, __func__);
1029 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1032 static void rec_special_MTLO(struct lightrec_cstate *state,
1033 const struct block *block, u16 offset)
1035 union code c = block->opcode_list[offset].c;
1037 _jit_name(block->_jit, __func__);
1038 rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1041 static void call_to_c_wrapper(struct lightrec_cstate *state,
1042 const struct block *block, u32 arg,
1043 enum c_wrappers wrapper)
1045 struct regcache *reg_cache = state->reg_cache;
1046 jit_state_t *_jit = block->_jit;
1049 /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1050 tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1052 tmp = lightrec_get_reg_with_value(reg_cache,
1053 (intptr_t) state->state->wrappers_eps[wrapper]);
1055 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1056 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1057 offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1059 lightrec_temp_set_value(reg_cache, tmp,
1060 (intptr_t) state->state->wrappers_eps[wrapper]);
1063 lightrec_free_reg(reg_cache, tmp2);
1066 /* On MIPS, register t9 is always used as the target register for JALR.
1067 * Therefore if it does not contain the target address we must
1070 lightrec_unload_reg(reg_cache, _jit, _T9);
1076 lightrec_regcache_mark_live(reg_cache, _jit);
1079 lightrec_free_reg(reg_cache, tmp);
1080 lightrec_regcache_mark_live(reg_cache, _jit);
1083 static void rec_io(struct lightrec_cstate *state,
1084 const struct block *block, u16 offset,
1085 bool load_rt, bool read_rt)
1087 struct regcache *reg_cache = state->reg_cache;
1088 jit_state_t *_jit = block->_jit;
1089 union code c = block->opcode_list[offset].c;
1090 u32 flags = block->opcode_list[offset].flags;
1091 bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1094 jit_note(__FILE__, __LINE__);
1096 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1098 if (read_rt && likely(c.i.rt))
1099 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1101 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1104 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1106 lut_entry = lightrec_get_lut_entry(block);
1107 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1108 C_WRAPPER_RW_GENERIC);
1112 static u32 rec_ram_mask(struct lightrec_state *state)
1114 return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1117 static u32 rec_io_mask(const struct lightrec_state *state)
1119 u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1121 return GENMASK(31 - clz32(length - 1), 0);
1124 static void rec_store_memory(struct lightrec_cstate *cstate,
1125 const struct block *block,
1126 u16 offset, jit_code_t code,
1127 jit_code_t swap_code,
1128 uintptr_t addr_offset, u32 addr_mask,
1131 const struct lightrec_state *state = cstate->state;
1132 struct regcache *reg_cache = cstate->reg_cache;
1133 struct opcode *op = &block->opcode_list[offset];
1134 jit_state_t *_jit = block->_jit;
1135 union code c = op->c;
1136 u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1137 s16 imm = (s16)c.i.imm;
1138 s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1139 s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1140 bool no_mask = op_flag_no_mask(op->flags);
1141 bool add_imm = c.i.imm &&
1142 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1143 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1144 bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1145 bool swc2 = c.i.op == OP_SWC2;
1146 u8 in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1148 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1149 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1151 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1156 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1164 jit_andi(tmp, addr_reg, addr_mask);
1169 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1170 jit_addi(tmp2, addr_reg, addr_offset);
1173 addr_reg2 = addr_reg;
1176 if (is_big_endian() && swap_code && in_reg) {
1177 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1179 jit_new_node_ww(swap_code, tmp3, rt);
1180 jit_new_node_www(code, imm, addr_reg2, tmp3);
1182 lightrec_free_reg(reg_cache, tmp3);
1184 jit_new_node_www(code, imm, addr_reg2, rt);
1187 lightrec_free_reg(reg_cache, rt);
1190 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1192 if (c.i.op != OP_SW) {
1193 jit_andi(tmp, addr_reg, ~3);
1197 if (!lut_is_32bit(state)) {
1198 jit_lshi(tmp, addr_reg, 1);
1202 if (addr_reg == rs && c.i.rs == 0) {
1203 addr_reg = LIGHTREC_REG_STATE;
1205 jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1209 if (lut_is_32bit(state))
1210 jit_stxi_i(lut_offt, addr_reg, tmp3);
1212 jit_stxi(lut_offt, addr_reg, tmp3);
1214 lightrec_free_reg(reg_cache, tmp3);
1218 lightrec_free_reg(reg_cache, tmp2);
1220 lightrec_free_reg(reg_cache, tmp);
1221 lightrec_free_reg(reg_cache, rs);
1224 static void rec_store_ram(struct lightrec_cstate *cstate,
1225 const struct block *block,
1226 u16 offset, jit_code_t code,
1227 jit_code_t swap_code, bool invalidate)
1229 struct lightrec_state *state = cstate->state;
1231 _jit_note(block->_jit, __FILE__, __LINE__);
1233 return rec_store_memory(cstate, block, offset, code, swap_code,
1234 state->offset_ram, rec_ram_mask(state),
1238 static void rec_store_scratch(struct lightrec_cstate *cstate,
1239 const struct block *block, u16 offset,
1240 jit_code_t code, jit_code_t swap_code)
1242 _jit_note(block->_jit, __FILE__, __LINE__);
1244 return rec_store_memory(cstate, block, offset, code, swap_code,
1245 cstate->state->offset_scratch,
1249 static void rec_store_io(struct lightrec_cstate *cstate,
1250 const struct block *block, u16 offset,
1251 jit_code_t code, jit_code_t swap_code)
1253 _jit_note(block->_jit, __FILE__, __LINE__);
1255 return rec_store_memory(cstate, block, offset, code, swap_code,
1256 cstate->state->offset_io,
1257 rec_io_mask(cstate->state), false);
1260 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1261 const struct block *block,
1262 u16 offset, jit_code_t code,
1263 jit_code_t swap_code)
1265 struct lightrec_state *state = cstate->state;
1266 struct regcache *reg_cache = cstate->reg_cache;
1267 union code c = block->opcode_list[offset].c;
1268 jit_state_t *_jit = block->_jit;
1269 jit_node_t *to_not_ram, *to_end;
1270 bool swc2 = c.i.op == OP_SWC2;
1271 u8 tmp, tmp2, rs, rt, in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1274 jit_note(__FILE__, __LINE__);
1275 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1276 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1278 if (state->offset_ram || state->offset_scratch)
1279 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1281 /* Convert to KUNSEG and avoid RAM mirrors */
1282 if (state->mirrors_mapped) {
1284 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1285 } else if (c.i.imm) {
1287 jit_addi(tmp, rs, (s16)c.i.imm);
1288 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1291 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1294 lightrec_free_reg(reg_cache, rs);
1296 if (state->offset_ram != state->offset_scratch) {
1297 to_not_ram = jit_bmsi(tmp, BIT(28));
1299 jit_movi(tmp2, state->offset_ram);
1302 jit_patch(to_not_ram);
1304 jit_movi(tmp2, state->offset_scratch);
1306 } else if (state->offset_ram) {
1307 jit_movi(tmp2, state->offset_ram);
1310 if (state->offset_ram || state->offset_scratch) {
1311 jit_addr(tmp, tmp, tmp2);
1312 lightrec_free_reg(reg_cache, tmp2);
1315 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1317 if (is_big_endian() && swap_code && in_reg) {
1318 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1320 jit_new_node_ww(swap_code, tmp2, rt);
1321 jit_new_node_www(code, imm, tmp, tmp2);
1323 lightrec_free_reg(reg_cache, tmp2);
1325 jit_new_node_www(code, imm, tmp, rt);
1328 lightrec_free_reg(reg_cache, rt);
1329 lightrec_free_reg(reg_cache, tmp);
1332 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1333 u16 offset, jit_code_t code, jit_code_t swap_code)
1335 struct lightrec_state *state = cstate->state;
1336 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1337 struct regcache *reg_cache = cstate->reg_cache;
1338 union code c = block->opcode_list[offset].c;
1339 jit_state_t *_jit = block->_jit;
1340 jit_node_t *to_not_ram, *to_end;
1341 bool swc2 = c.i.op == OP_SWC2;
1342 u8 tmp, tmp2, tmp3, masked_reg, rs, rt;
1343 u8 in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1345 jit_note(__FILE__, __LINE__);
1347 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1348 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1349 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1351 /* Convert to KUNSEG and avoid RAM mirrors */
1353 jit_addi(tmp2, rs, (s16)c.i.imm);
1354 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1356 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1359 lightrec_free_reg(reg_cache, rs);
1360 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1362 if (state->offset_ram != state->offset_scratch) {
1363 to_not_ram = jit_bgti(tmp2, ram_size);
1366 jit_lti_u(tmp, tmp2, ram_size);
1367 jit_movnr(tmp, tmp2, tmp);
1371 /* Compute the offset to the code LUT */
1372 if (c.i.op == OP_SW)
1373 jit_andi(tmp, masked_reg, RAM_SIZE - 1);
1375 jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3);
1377 if (!lut_is_32bit(state))
1378 jit_lshi(tmp, tmp, 1);
1379 jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1381 /* Write NULL to the code LUT to invalidate any block that's there */
1382 if (lut_is_32bit(state))
1383 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1385 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1387 if (state->offset_ram != state->offset_scratch) {
1388 jit_movi(tmp, state->offset_ram);
1391 jit_patch(to_not_ram);
1394 if (state->offset_ram || state->offset_scratch)
1395 jit_movi(tmp, state->offset_scratch);
1397 if (state->offset_ram != state->offset_scratch)
1400 if (state->offset_ram || state->offset_scratch)
1401 jit_addr(tmp2, tmp2, tmp);
1403 lightrec_free_reg(reg_cache, tmp);
1404 lightrec_free_reg(reg_cache, tmp3);
1406 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1408 if (is_big_endian() && swap_code && in_reg) {
1409 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1411 jit_new_node_ww(swap_code, tmp, rt);
1412 jit_new_node_www(code, 0, tmp2, tmp);
1414 lightrec_free_reg(reg_cache, tmp);
1416 jit_new_node_www(code, 0, tmp2, rt);
1419 lightrec_free_reg(reg_cache, rt);
1420 lightrec_free_reg(reg_cache, tmp2);
1423 static void rec_store(struct lightrec_cstate *state,
1424 const struct block *block, u16 offset,
1425 jit_code_t code, jit_code_t swap_code)
1427 u32 flags = block->opcode_list[offset].flags;
1428 u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1429 bool no_invalidate = op_flag_no_invalidate(flags) ||
1430 state->state->invalidate_from_dma_only;
1431 union code c = block->opcode_list[offset].c;
1432 bool is_swc2 = c.i.op == OP_SWC2;
1436 case LIGHTREC_IO_RAM:
1437 case LIGHTREC_IO_SCRATCH:
1438 case LIGHTREC_IO_DIRECT:
1439 case LIGHTREC_IO_DIRECT_HW:
1440 rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_CP2_TEMP);
1448 case LIGHTREC_IO_RAM:
1449 rec_store_ram(state, block, offset, code,
1450 swap_code, !no_invalidate);
1452 case LIGHTREC_IO_SCRATCH:
1453 rec_store_scratch(state, block, offset, code, swap_code);
1455 case LIGHTREC_IO_DIRECT:
1456 if (no_invalidate) {
1457 rec_store_direct_no_invalidate(state, block, offset,
1460 rec_store_direct(state, block, offset, code, swap_code);
1463 case LIGHTREC_IO_DIRECT_HW:
1464 rec_store_io(state, block, offset, code, swap_code);
1467 rec_io(state, block, offset, true, false);
1472 lightrec_discard_reg_if_loaded(state->reg_cache, REG_CP2_TEMP);
1475 static void rec_SB(struct lightrec_cstate *state,
1476 const struct block *block, u16 offset)
1478 _jit_name(block->_jit, __func__);
1479 rec_store(state, block, offset, jit_code_stxi_c, 0);
1482 static void rec_SH(struct lightrec_cstate *state,
1483 const struct block *block, u16 offset)
1485 _jit_name(block->_jit, __func__);
1486 rec_store(state, block, offset,
1487 jit_code_stxi_s, jit_code_bswapr_us);
1490 static void rec_SW(struct lightrec_cstate *state,
1491 const struct block *block, u16 offset)
1494 union code c = block->opcode_list[offset].c;
1496 _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
1497 rec_store(state, block, offset,
1498 jit_code_stxi_i, jit_code_bswapr_ui);
1501 static void rec_SWL(struct lightrec_cstate *state,
1502 const struct block *block, u16 offset)
1504 _jit_name(block->_jit, __func__);
1505 rec_io(state, block, offset, true, false);
1508 static void rec_SWR(struct lightrec_cstate *state,
1509 const struct block *block, u16 offset)
1511 _jit_name(block->_jit, __func__);
1512 rec_io(state, block, offset, true, false);
1515 static void rec_load_memory(struct lightrec_cstate *cstate,
1516 const struct block *block, u16 offset,
1517 jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1518 uintptr_t addr_offset, u32 addr_mask)
1520 struct regcache *reg_cache = cstate->reg_cache;
1521 struct opcode *op = &block->opcode_list[offset];
1522 jit_state_t *_jit = block->_jit;
1523 u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
1524 bool no_mask = op_flag_no_mask(op->flags);
1525 union code c = op->c;
1528 if (c.i.op == OP_LWC2)
1529 out_reg = REG_CP2_TEMP;
1538 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1539 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1541 if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1542 jit_addi(rt, rs, (s16)c.i.imm);
1551 jit_andi(rt, addr_reg, addr_mask);
1556 jit_addi(rt, addr_reg, addr_offset);
1560 jit_new_node_www(code, rt, addr_reg, imm);
1562 if (is_big_endian() && swap_code) {
1563 jit_new_node_ww(swap_code, rt, rt);
1565 if (c.i.op == OP_LH)
1567 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1571 lightrec_free_reg(reg_cache, rs);
1572 lightrec_free_reg(reg_cache, rt);
1575 static void rec_load_ram(struct lightrec_cstate *cstate,
1576 const struct block *block, u16 offset,
1577 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1579 _jit_note(block->_jit, __FILE__, __LINE__);
1581 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1582 cstate->state->offset_ram, rec_ram_mask(cstate->state));
1585 static void rec_load_bios(struct lightrec_cstate *cstate,
1586 const struct block *block, u16 offset,
1587 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1589 _jit_note(block->_jit, __FILE__, __LINE__);
1591 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1592 cstate->state->offset_bios, 0x1fffffff);
1595 static void rec_load_scratch(struct lightrec_cstate *cstate,
1596 const struct block *block, u16 offset,
1597 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1599 _jit_note(block->_jit, __FILE__, __LINE__);
1601 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1602 cstate->state->offset_scratch, 0x1fffffff);
1605 static void rec_load_io(struct lightrec_cstate *cstate,
1606 const struct block *block, u16 offset,
1607 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1609 _jit_note(block->_jit, __FILE__, __LINE__);
1611 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1612 cstate->state->offset_io, rec_io_mask(cstate->state));
1615 static void rec_load_direct(struct lightrec_cstate *cstate,
1616 const struct block *block, u16 offset,
1617 jit_code_t code, jit_code_t swap_code,
1620 struct lightrec_state *state = cstate->state;
1621 struct regcache *reg_cache = cstate->reg_cache;
1622 union code c = block->opcode_list[offset].c;
1623 jit_state_t *_jit = block->_jit;
1624 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1625 u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
1628 if (c.i.op == OP_LWC2)
1629 out_reg = REG_CP2_TEMP;
1638 jit_note(__FILE__, __LINE__);
1639 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1640 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1642 if ((state->offset_ram == state->offset_bios &&
1643 state->offset_ram == state->offset_scratch &&
1644 state->mirrors_mapped) || !c.i.imm) {
1648 jit_addi(rt, rs, (s16)c.i.imm);
1652 if (c.i.rs != c.i.rt)
1653 lightrec_free_reg(reg_cache, rs);
1656 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1658 if (state->offset_ram == state->offset_bios &&
1659 state->offset_ram == state->offset_scratch) {
1660 if (!state->mirrors_mapped) {
1661 jit_andi(tmp, addr_reg, BIT(28));
1662 jit_rshi_u(tmp, tmp, 28 - 22);
1663 jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1664 jit_andr(rt, addr_reg, tmp);
1666 jit_andi(rt, addr_reg, 0x1fffffff);
1669 if (state->offset_ram)
1670 jit_movi(tmp, state->offset_ram);
1672 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1674 /* Convert to KUNSEG and avoid RAM mirrors */
1675 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1677 if (state->offset_ram)
1678 jit_movi(tmp, state->offset_ram);
1682 jit_patch(to_not_ram);
1684 if (state->offset_bios != state->offset_scratch)
1685 to_not_bios = jit_bmci(addr_reg, BIT(22));
1687 /* Convert to KUNSEG */
1688 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1690 jit_movi(tmp, state->offset_bios);
1692 if (state->offset_bios != state->offset_scratch) {
1695 jit_patch(to_not_bios);
1697 /* Convert to KUNSEG */
1698 jit_andi(rt, addr_reg, 0x1f800fff);
1700 if (state->offset_scratch)
1701 jit_movi(tmp, state->offset_scratch);
1709 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1710 jit_addr(rt, rt, tmp);
1712 jit_new_node_www(code, rt, rt, imm);
1714 if (is_big_endian() && swap_code) {
1715 jit_new_node_ww(swap_code, rt, rt);
1717 if (c.i.op == OP_LH)
1719 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1723 lightrec_free_reg(reg_cache, addr_reg);
1724 lightrec_free_reg(reg_cache, rt);
1725 lightrec_free_reg(reg_cache, tmp);
1728 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1729 u16 offset, jit_code_t code, jit_code_t swap_code,
1732 const struct opcode *op = &block->opcode_list[offset];
1733 u32 flags = op->flags;
1735 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1736 case LIGHTREC_IO_RAM:
1737 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1739 case LIGHTREC_IO_BIOS:
1740 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1742 case LIGHTREC_IO_SCRATCH:
1743 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1745 case LIGHTREC_IO_DIRECT_HW:
1746 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1748 case LIGHTREC_IO_DIRECT:
1749 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1752 rec_io(state, block, offset, false, true);
1756 if (op->i.op == OP_LWC2) {
1757 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_CP2_TEMP);
1758 lightrec_discard_reg_if_loaded(state->reg_cache, REG_CP2_TEMP);
1762 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1764 _jit_name(block->_jit, __func__);
1765 rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1768 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1770 _jit_name(block->_jit, __func__);
1771 rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1774 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1776 jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1778 _jit_name(block->_jit, __func__);
1779 rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1782 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1784 _jit_name(block->_jit, __func__);
1785 rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1788 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1790 _jit_name(block->_jit, __func__);
1791 rec_io(state, block, offset, true, true);
1794 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1796 _jit_name(block->_jit, __func__);
1797 rec_io(state, block, offset, true, true);
1800 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1802 union code c = block->opcode_list[offset].c;
1805 if (is_big_endian() && __WORDSIZE == 64)
1806 code = jit_code_ldxi_ui;
1808 code = jit_code_ldxi_i;
1810 _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
1811 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1814 static void rec_break_syscall(struct lightrec_cstate *state,
1815 const struct block *block, u16 offset,
1818 struct regcache *reg_cache = state->reg_cache;
1819 jit_state_t *_jit = block->_jit;
1822 _jit_note(block->_jit, __FILE__, __LINE__);
1824 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1826 jit_movi(tmp, exit_code);
1827 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1828 LIGHTREC_REG_STATE, tmp);
1830 lightrec_free_reg(reg_cache, tmp);
1832 /* TODO: the return address should be "pc - 4" if we're a delay slot */
1833 lightrec_emit_end_of_block(state, block, offset, -1,
1834 get_ds_pc(block, offset, 0),
1838 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1839 const struct block *block, u16 offset)
1841 _jit_name(block->_jit, __func__);
1842 rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1845 static void rec_special_BREAK(struct lightrec_cstate *state,
1846 const struct block *block, u16 offset)
1848 _jit_name(block->_jit, __func__);
1849 rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1852 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1854 struct regcache *reg_cache = state->reg_cache;
1855 union code c = block->opcode_list[offset].c;
1856 jit_state_t *_jit = block->_jit;
1858 jit_note(__FILE__, __LINE__);
1860 if (c.i.op != OP_SWC2)
1861 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1863 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1866 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1868 struct regcache *reg_cache = state->reg_cache;
1869 union code c = block->opcode_list[offset].c;
1870 jit_state_t *_jit = block->_jit;
1872 jit_note(__FILE__, __LINE__);
1873 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1874 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1876 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1878 if (c.i.op == OP_CP0 &&
1879 !op_flag_no_ds(block->opcode_list[offset].flags) &&
1880 (c.r.rd == 12 || c.r.rd == 13))
1881 lightrec_emit_end_of_block(state, block, offset, -1,
1882 get_ds_pc(block, offset, 1),
1887 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1889 struct regcache *reg_cache = state->reg_cache;
1890 union code c = block->opcode_list[offset].c;
1891 jit_state_t *_jit = block->_jit;
1894 jit_note(__FILE__, __LINE__);
1896 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1898 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1899 offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1901 lightrec_free_reg(reg_cache, rt);
1904 static bool block_in_bios(const struct lightrec_cstate *state,
1905 const struct block *block)
1907 const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1908 u32 pc = kunseg(block->pc);
1910 return pc >= bios->pc && pc < bios->pc + bios->length;
1914 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1916 struct regcache *reg_cache = state->reg_cache;
1917 const union code c = block->opcode_list[offset].c;
1918 jit_state_t *_jit = block->_jit;
1919 u8 rt, tmp = 0, tmp2, status;
1922 jit_note(__FILE__, __LINE__);
1930 /* Those registers are read-only */
1936 if (block_in_bios(state, block) && c.r.rd == 12) {
1937 /* If we are running code from the BIOS, handle writes to the
1938 * Status register in C. BIOS code may toggle bit 16 which will
1939 * map/unmap the RAM, while game code cannot do that. */
1940 rec_mtc(state, block, offset);
1944 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1947 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1948 LIGHTREC_REG_STATE, rt);
1951 if (c.r.rd == 12 || c.r.rd == 13) {
1952 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1953 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1954 offsetof(struct lightrec_state, regs.cp0[13]));
1956 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1961 } else if (c.r.rd == 13) {
1962 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1963 jit_andi(tmp2, rt, 0x0300);
1964 jit_ori(tmp, tmp, 0x0300);
1965 jit_xori(tmp, tmp, 0x0300);
1966 jit_orr(tmp, tmp, tmp2);
1967 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1968 offsetof(struct lightrec_state, regs.cp0[12]));
1969 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1970 LIGHTREC_REG_STATE, tmp);
1974 if (c.r.rd == 12 || c.r.rd == 13) {
1975 /* Exit dynarec in case there's a software interrupt.
1976 * exit_flags = !!(status & tmp & 0x0300) & status; */
1977 jit_andr(tmp, tmp, status);
1978 jit_andi(tmp, tmp, 0x0300);
1979 jit_nei(tmp, tmp, 0);
1980 jit_andr(tmp, tmp, status);
1984 /* Exit dynarec in case we unmask a hardware interrupt.
1985 * exit_flags = !(~status & 0x401) */
1987 jit_comr(tmp2, status);
1988 jit_andi(tmp2, tmp2, 0x401);
1989 jit_eqi(tmp2, tmp2, 0);
1990 jit_orr(tmp, tmp, tmp2);
1993 lightrec_free_reg(reg_cache, rt);
1995 if (c.r.rd == 12 || c.r.rd == 13) {
1996 to_end = jit_beqi(tmp, 0);
1998 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1999 offsetof(struct lightrec_state, target_cycle));
2000 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2001 jit_movi(LIGHTREC_REG_CYCLE, 0);
2002 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2003 LIGHTREC_REG_STATE, tmp2);
2004 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2005 LIGHTREC_REG_STATE, tmp2);
2011 if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
2012 (c.r.rd == 12 || c.r.rd == 13)) {
2013 state->cycles += lightrec_cycles_of_opcode(c);
2014 lightrec_emit_eob(state, block, offset + 1);
2018 static void rec_cp0_MFC0(struct lightrec_cstate *state,
2019 const struct block *block, u16 offset)
2021 _jit_name(block->_jit, __func__);
2022 rec_mfc0(state, block, offset);
2025 static void rec_cp0_CFC0(struct lightrec_cstate *state,
2026 const struct block *block, u16 offset)
2028 _jit_name(block->_jit, __func__);
2029 rec_mfc0(state, block, offset);
2032 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2033 const struct block *block, u16 offset)
2035 _jit_name(block->_jit, __func__);
2036 rec_mtc0(state, block, offset);
2039 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2040 const struct block *block, u16 offset)
2042 _jit_name(block->_jit, __func__);
2043 rec_mtc0(state, block, offset);
2046 static unsigned int cp2d_i_offset(u8 reg)
2048 return offsetof(struct lightrec_state, regs.cp2d[reg]);
2051 static unsigned int cp2d_s_offset(u8 reg)
2053 return cp2d_i_offset(reg) + is_big_endian() * 2;
2056 static unsigned int cp2c_i_offset(u8 reg)
2058 return offsetof(struct lightrec_state, regs.cp2c[reg]);
2061 static unsigned int cp2c_s_offset(u8 reg)
2063 return cp2c_i_offset(reg) + is_big_endian() * 2;
2066 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2067 const struct block *block, u16 offset,
2070 struct regcache *reg_cache = state->reg_cache;
2071 jit_state_t *_jit = block->_jit;
2072 const u32 zext_regs = 0x300f0080;
2073 u8 rt, tmp, tmp2, tmp3, out, flags;
2076 _jit_name(block->_jit, __func__);
2078 if (state->state->ops.cop2_notify) {
2079 /* We must call cop2_notify, handle that in C. */
2080 rec_mfc(state, block, offset);
2084 flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2085 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2098 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2105 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2109 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2110 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2111 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2113 for (i = 0; i < 3; i++) {
2114 out = i == 0 ? rt : tmp;
2116 jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2117 jit_movi(tmp2, 0x1f);
2118 jit_rshi(out, tmp, 7);
2120 jit_ltr(tmp3, tmp2, out);
2121 jit_movnr(out, tmp2, tmp3);
2123 jit_gei(tmp2, out, 0);
2124 jit_movzr(out, tmp2, tmp2);
2127 jit_lshi(tmp, tmp, 5 * i);
2128 jit_orr(rt, rt, tmp);
2133 lightrec_free_reg(reg_cache, tmp);
2134 lightrec_free_reg(reg_cache, tmp2);
2135 lightrec_free_reg(reg_cache, tmp3);
2138 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2142 lightrec_free_reg(reg_cache, rt);
2145 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2146 const struct block *block, u16 offset)
2148 const union code c = block->opcode_list[offset].c;
2150 rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2153 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2154 const struct block *block, u16 offset)
2156 struct regcache *reg_cache = state->reg_cache;
2157 const union code c = block->opcode_list[offset].c;
2158 jit_state_t *_jit = block->_jit;
2161 _jit_name(block->_jit, __func__);
2163 if (state->state->ops.cop2_notify) {
2164 /* We must call cop2_notify, handle that in C. */
2165 rec_mfc(state, block, offset);
2177 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2178 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2181 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2182 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2186 lightrec_free_reg(reg_cache, rt);
2189 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2190 const struct block *block, u16 offset,
2193 struct regcache *reg_cache = state->reg_cache;
2194 jit_state_t *_jit = block->_jit;
2195 jit_node_t *loop, *to_loop;
2196 u8 rt, tmp, tmp2, flags = 0;
2198 _jit_name(block->_jit, __func__);
2200 if (state->state->ops.cop2_notify) {
2201 /* We must call cop2_notify, handle that in C. */
2202 rec_mtc(state, block, offset);
2212 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
2216 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2217 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2219 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2220 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2222 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2223 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2224 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2226 lightrec_free_reg(reg_cache, tmp);
2227 lightrec_free_reg(reg_cache, tmp2);
2230 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2232 jit_lshi(tmp, rt, 7);
2233 jit_andi(tmp, tmp, 0xf80);
2234 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2236 jit_lshi(tmp, rt, 2);
2237 jit_andi(tmp, tmp, 0xf80);
2238 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2240 jit_rshi(tmp, rt, 3);
2241 jit_andi(tmp, tmp, 0xf80);
2242 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2244 lightrec_free_reg(reg_cache, tmp);
2247 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2248 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2250 /* if (rt < 0) rt = ~rt; */
2251 jit_rshi(tmp, rt, 31);
2252 jit_xorr(tmp, rt, tmp);
2254 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2255 jit_lshi(tmp, tmp, 1);
2258 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2260 jit_subi(tmp2, tmp2, 1);
2261 jit_rshi_u(tmp, tmp, 1);
2262 to_loop = jit_bnei(tmp, 0);
2264 jit_patch_at(to_loop, loop);
2266 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2267 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2269 lightrec_free_reg(reg_cache, tmp);
2270 lightrec_free_reg(reg_cache, tmp2);
2273 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
2277 lightrec_free_reg(reg_cache, rt);
2280 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2281 const struct block *block, u16 offset)
2283 const union code c = block->opcode_list[offset].c;
2285 rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2288 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2289 const struct block *block, u16 offset)
2291 struct regcache *reg_cache = state->reg_cache;
2292 const union code c = block->opcode_list[offset].c;
2293 jit_state_t *_jit = block->_jit;
2296 _jit_name(block->_jit, __func__);
2298 if (state->state->ops.cop2_notify) {
2299 /* We must call cop2_notify, handle that in C. */
2300 rec_mtc(state, block, offset);
2304 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2314 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2317 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2318 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2320 jit_andi(tmp, rt, 0x7f87e000);
2321 jit_nei(tmp, tmp, 0);
2322 jit_lshi(tmp, tmp, 31);
2324 jit_andi(tmp2, rt, 0x7ffff000);
2325 jit_orr(tmp, tmp2, tmp);
2327 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2329 lightrec_free_reg(reg_cache, tmp);
2330 lightrec_free_reg(reg_cache, tmp2);
2334 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2337 lightrec_free_reg(reg_cache, rt);
2340 static void rec_cp0_RFE(struct lightrec_cstate *state,
2341 const struct block *block, u16 offset)
2343 struct regcache *reg_cache = state->reg_cache;
2344 jit_state_t *_jit = block->_jit;
2348 jit_note(__FILE__, __LINE__);
2350 status = lightrec_alloc_reg_temp(reg_cache, _jit);
2351 jit_ldxi_i(status, LIGHTREC_REG_STATE,
2352 offsetof(struct lightrec_state, regs.cp0[12]));
2354 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2356 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2357 jit_rshi(tmp, status, 2);
2358 jit_andi(tmp, tmp, 0xf);
2359 jit_andi(status, status, ~0xful);
2360 jit_orr(status, status, tmp);
2362 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2363 offsetof(struct lightrec_state, regs.cp0[13]));
2364 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2365 LIGHTREC_REG_STATE, status);
2367 /* Exit dynarec in case there's a software interrupt.
2368 * exit_flags = !!(status & cause & 0x0300) & status; */
2369 jit_andr(tmp, tmp, status);
2370 jit_andi(tmp, tmp, 0x0300);
2371 jit_nei(tmp, tmp, 0);
2372 jit_andr(tmp, tmp, status);
2373 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2374 LIGHTREC_REG_STATE, tmp);
2376 lightrec_free_reg(reg_cache, status);
2377 lightrec_free_reg(reg_cache, tmp);
2380 static void rec_CP(struct lightrec_cstate *state,
2381 const struct block *block, u16 offset)
2383 union code c = block->opcode_list[offset].c;
2384 jit_state_t *_jit = block->_jit;
2387 jit_note(__FILE__, __LINE__);
2389 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2392 static void rec_meta_MOV(struct lightrec_cstate *state,
2393 const struct block *block, u16 offset)
2395 struct regcache *reg_cache = state->reg_cache;
2396 const struct opcode *op = &block->opcode_list[offset];
2397 union code c = op->c;
2398 jit_state_t *_jit = block->_jit;
2402 _jit_name(block->_jit, __func__);
2403 jit_note(__FILE__, __LINE__);
2405 unload_rd = OPT_EARLY_UNLOAD
2406 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2408 if (c.r.rs || unload_rd)
2409 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2412 /* If the destination register will be unloaded right after the
2413 * MOV meta-opcode, we don't actually need to write any host
2414 * register - we can just store the source register directly to
2415 * the register cache, at the offset corresponding to the
2416 * destination register. */
2417 lightrec_discard_reg_if_loaded(reg_cache, c.r.rd);
2419 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
2420 + c.r.rd << 2, LIGHTREC_REG_STATE, rs);
2422 lightrec_free_reg(reg_cache, rs);
2424 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2431 lightrec_free_reg(reg_cache, rd);
2434 if (c.r.rs || unload_rd)
2435 lightrec_free_reg(reg_cache, rs);
2438 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2439 const struct block *block,
2442 struct regcache *reg_cache = state->reg_cache;
2443 union code c = block->opcode_list[offset].c;
2444 jit_state_t *_jit = block->_jit;
2447 _jit_name(block->_jit, __func__);
2448 jit_note(__FILE__, __LINE__);
2450 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2451 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2453 if (c.i.op == OP_META_EXTC)
2458 lightrec_free_reg(reg_cache, rs);
2459 lightrec_free_reg(reg_cache, rt);
2462 static void rec_meta_MULT2(struct lightrec_cstate *state,
2463 const struct block *block,
2466 struct regcache *reg_cache = state->reg_cache;
2467 union code c = block->opcode_list[offset].c;
2468 jit_state_t *_jit = block->_jit;
2469 u8 reg_lo = get_mult_div_lo(c);
2470 u8 reg_hi = get_mult_div_hi(c);
2471 u32 flags = block->opcode_list[offset].flags;
2472 bool is_signed = c.i.op == OP_META_MULT2;
2473 u8 rs, lo, hi, rflags = 0, hiflags = 0;
2476 if (!op_flag_no_hi(flags) && c.r.op < 32) {
2477 rflags = is_signed ? REG_EXT : REG_ZEXT;
2478 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2481 _jit_name(block->_jit, __func__);
2482 jit_note(__FILE__, __LINE__);
2484 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2487 * We must handle the case where one of the output registers is our rs
2488 * input register. Thanksfully, computing LO/HI can be done in any
2489 * order. Here, we make sure that the computation that overwrites the
2490 * input register is always performed last.
2492 for (i = 0; i < 2; i++) {
2493 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2494 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2497 jit_lshi(lo, rs, c.r.op);
2501 lightrec_free_reg(reg_cache, lo);
2505 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2506 hi = lightrec_alloc_reg_out(reg_cache, _jit,
2510 jit_lshi(hi, rs, c.r.op - 32);
2512 jit_rshi(hi, rs, 32 - c.r.op);
2514 jit_rshi_u(hi, rs, 32 - c.r.op);
2516 lightrec_free_reg(reg_cache, hi);
2520 lightrec_free_reg(reg_cache, rs);
2522 _jit_name(block->_jit, __func__);
2523 jit_note(__FILE__, __LINE__);
2526 static const lightrec_rec_func_t rec_standard[64] = {
2527 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2528 [OP_SPECIAL] = rec_SPECIAL,
2529 [OP_REGIMM] = rec_REGIMM,
2534 [OP_BLEZ] = rec_BLEZ,
2535 [OP_BGTZ] = rec_BGTZ,
2536 [OP_ADDI] = rec_ADDI,
2537 [OP_ADDIU] = rec_ADDIU,
2538 [OP_SLTI] = rec_SLTI,
2539 [OP_SLTIU] = rec_SLTIU,
2540 [OP_ANDI] = rec_ANDI,
2542 [OP_XORI] = rec_XORI,
2561 [OP_META_MOV] = rec_meta_MOV,
2562 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
2563 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
2564 [OP_META_MULT2] = rec_meta_MULT2,
2565 [OP_META_MULTU2] = rec_meta_MULT2,
2568 static const lightrec_rec_func_t rec_special[64] = {
2569 SET_DEFAULT_ELM(rec_special, unknown_opcode),
2570 [OP_SPECIAL_SLL] = rec_special_SLL,
2571 [OP_SPECIAL_SRL] = rec_special_SRL,
2572 [OP_SPECIAL_SRA] = rec_special_SRA,
2573 [OP_SPECIAL_SLLV] = rec_special_SLLV,
2574 [OP_SPECIAL_SRLV] = rec_special_SRLV,
2575 [OP_SPECIAL_SRAV] = rec_special_SRAV,
2576 [OP_SPECIAL_JR] = rec_special_JR,
2577 [OP_SPECIAL_JALR] = rec_special_JALR,
2578 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
2579 [OP_SPECIAL_BREAK] = rec_special_BREAK,
2580 [OP_SPECIAL_MFHI] = rec_special_MFHI,
2581 [OP_SPECIAL_MTHI] = rec_special_MTHI,
2582 [OP_SPECIAL_MFLO] = rec_special_MFLO,
2583 [OP_SPECIAL_MTLO] = rec_special_MTLO,
2584 [OP_SPECIAL_MULT] = rec_special_MULT,
2585 [OP_SPECIAL_MULTU] = rec_special_MULTU,
2586 [OP_SPECIAL_DIV] = rec_special_DIV,
2587 [OP_SPECIAL_DIVU] = rec_special_DIVU,
2588 [OP_SPECIAL_ADD] = rec_special_ADD,
2589 [OP_SPECIAL_ADDU] = rec_special_ADDU,
2590 [OP_SPECIAL_SUB] = rec_special_SUB,
2591 [OP_SPECIAL_SUBU] = rec_special_SUBU,
2592 [OP_SPECIAL_AND] = rec_special_AND,
2593 [OP_SPECIAL_OR] = rec_special_OR,
2594 [OP_SPECIAL_XOR] = rec_special_XOR,
2595 [OP_SPECIAL_NOR] = rec_special_NOR,
2596 [OP_SPECIAL_SLT] = rec_special_SLT,
2597 [OP_SPECIAL_SLTU] = rec_special_SLTU,
2600 static const lightrec_rec_func_t rec_regimm[64] = {
2601 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2602 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
2603 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
2604 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
2605 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
2608 static const lightrec_rec_func_t rec_cp0[64] = {
2609 SET_DEFAULT_ELM(rec_cp0, rec_CP),
2610 [OP_CP0_MFC0] = rec_cp0_MFC0,
2611 [OP_CP0_CFC0] = rec_cp0_CFC0,
2612 [OP_CP0_MTC0] = rec_cp0_MTC0,
2613 [OP_CP0_CTC0] = rec_cp0_CTC0,
2614 [OP_CP0_RFE] = rec_cp0_RFE,
2617 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2618 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2619 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
2620 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
2621 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
2622 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
2625 static void rec_SPECIAL(struct lightrec_cstate *state,
2626 const struct block *block, u16 offset)
2628 union code c = block->opcode_list[offset].c;
2629 lightrec_rec_func_t f = rec_special[c.r.op];
2631 if (!HAS_DEFAULT_ELM && unlikely(!f))
2632 unknown_opcode(state, block, offset);
2634 (*f)(state, block, offset);
2637 static void rec_REGIMM(struct lightrec_cstate *state,
2638 const struct block *block, u16 offset)
2640 union code c = block->opcode_list[offset].c;
2641 lightrec_rec_func_t f = rec_regimm[c.r.rt];
2643 if (!HAS_DEFAULT_ELM && unlikely(!f))
2644 unknown_opcode(state, block, offset);
2646 (*f)(state, block, offset);
2649 static void rec_CP0(struct lightrec_cstate *state,
2650 const struct block *block, u16 offset)
2652 union code c = block->opcode_list[offset].c;
2653 lightrec_rec_func_t f = rec_cp0[c.r.rs];
2655 if (!HAS_DEFAULT_ELM && unlikely(!f))
2656 rec_CP(state, block, offset);
2658 (*f)(state, block, offset);
2661 static void rec_CP2(struct lightrec_cstate *state,
2662 const struct block *block, u16 offset)
2664 union code c = block->opcode_list[offset].c;
2666 if (c.r.op == OP_CP2_BASIC) {
2667 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2669 if (HAS_DEFAULT_ELM || likely(f)) {
2670 (*f)(state, block, offset);
2675 rec_CP(state, block, offset);
2678 void lightrec_rec_opcode(struct lightrec_cstate *state,
2679 const struct block *block, u16 offset)
2681 struct regcache *reg_cache = state->reg_cache;
2682 struct lightrec_branch_target *target;
2683 const struct opcode *op = &block->opcode_list[offset];
2684 jit_state_t *_jit = block->_jit;
2685 lightrec_rec_func_t f;
2688 if (op_flag_sync(op->flags)) {
2690 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2693 lightrec_storeback_regs(reg_cache, _jit);
2694 lightrec_regcache_reset(reg_cache);
2696 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2697 target = &state->targets[state->nb_targets++];
2698 target->offset = offset;
2699 target->label = jit_indirect();
2702 if (likely(op->opcode)) {
2703 f = rec_standard[op->i.op];
2705 if (!HAS_DEFAULT_ELM && unlikely(!f))
2706 unknown_opcode(state, block, offset);
2708 (*f)(state, block, offset);
2711 if (OPT_EARLY_UNLOAD) {
2712 unload_offset = offset +
2713 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2715 lightrec_do_early_unload(state, block, unload_offset);