1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "disassembler.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
27 pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
28 block->opcode_list[offset].c.opcode,
29 block->pc + (offset << 2));
33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
35 /* Prevent jit_jmpi() from using our cycles register as a temporary */
36 jit_live(LIGHTREC_REG_CYCLE);
38 jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
42 const struct block *block, u16 offset,
43 s8 reg_new_pc, u32 imm, u8 ra_reg,
44 u32 link, bool update_cycles)
46 struct regcache *reg_cache = state->reg_cache;
47 jit_state_t *_jit = block->_jit;
48 const struct opcode *op = &block->opcode_list[offset],
49 *next = &block->opcode_list[offset + 1];
50 u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
52 jit_note(__FILE__, __LINE__);
55 /* Update the $ra register */
56 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
57 jit_movi(link_reg, link);
58 lightrec_free_reg(reg_cache, link_reg);
62 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
63 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
65 jit_movi(reg_new_pc, imm);
68 if (has_delay_slot(op->c) &&
69 !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
70 cycles += lightrec_cycles_of_opcode(next->c);
72 /* Recompile the delay slot */
74 lightrec_rec_opcode(state, block, offset + 1);
77 /* Clean the remaining registers */
78 lightrec_clean_regs(reg_cache, _jit);
80 jit_movr(JIT_V0, reg_new_pc);
82 if (cycles && update_cycles) {
83 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
84 pr_debug("EOB: %u cycles\n", cycles);
87 lightrec_jump_to_eob(state, _jit);
90 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
91 u16 offset, bool after_op)
93 struct regcache *reg_cache = state->reg_cache;
94 jit_state_t *_jit = block->_jit;
95 union code c = block->opcode_list[offset].c;
96 u32 cycles = state->cycles;
99 cycles += lightrec_cycles_of_opcode(c);
101 lightrec_clean_regs(reg_cache, _jit);
103 jit_movi(JIT_V0, block->pc + (offset << 2));
104 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
106 lightrec_jump_to_eob(state, _jit);
109 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
111 struct regcache *reg_cache = state->reg_cache;
112 jit_state_t *_jit = block->_jit;
113 const struct opcode *op = &block->opcode_list[offset];
116 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
117 lightrec_lock_reg(reg_cache, _jit, rs);
122 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
124 u8 rs = get_jr_jalr_reg(state, block, offset);
126 _jit_name(block->_jit, __func__);
127 lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
130 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
132 u8 rs = get_jr_jalr_reg(state, block, offset);
133 union code c = block->opcode_list[offset].c;
135 _jit_name(block->_jit, __func__);
136 lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
137 get_branch_pc(block, offset, 2), true);
140 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
142 union code c = block->opcode_list[offset].c;
144 _jit_name(block->_jit, __func__);
145 lightrec_emit_end_of_block(state, block, offset, -1,
146 (block->pc & 0xf0000000) | (c.j.imm << 2),
150 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
152 union code c = block->opcode_list[offset].c;
154 _jit_name(block->_jit, __func__);
155 lightrec_emit_end_of_block(state, block, offset, -1,
156 (block->pc & 0xf0000000) | (c.j.imm << 2),
157 31, get_branch_pc(block, offset, 2), true);
160 static void lightrec_do_early_unload(struct lightrec_cstate *state,
161 const struct block *block, u16 offset)
163 struct regcache *reg_cache = state->reg_cache;
164 const struct opcode *op = &block->opcode_list[offset];
165 jit_state_t *_jit = block->_jit;
171 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
172 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
173 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
176 for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
177 reg = reg_ops[i].reg;
179 switch (reg_ops[i].op) {
180 case LIGHTREC_REG_UNLOAD:
181 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
184 case LIGHTREC_REG_DISCARD:
185 lightrec_discard_reg_if_loaded(reg_cache, reg);
188 case LIGHTREC_REG_CLEAN:
189 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
197 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
198 jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
200 struct regcache *reg_cache = state->reg_cache;
201 struct native_register *regs_backup;
202 jit_state_t *_jit = block->_jit;
203 struct lightrec_branch *branch;
204 const struct opcode *op = &block->opcode_list[offset],
205 *next = &block->opcode_list[offset + 1];
208 bool is_forward = (s16)op->i.imm >= -1;
209 int op_cycles = lightrec_cycles_of_opcode(op->c);
210 u32 target_offset, cycles = state->cycles + op_cycles;
211 bool no_indirection = false;
214 jit_note(__FILE__, __LINE__);
216 if (!op_flag_no_ds(op->flags))
217 cycles += lightrec_cycles_of_opcode(next->c);
219 state->cycles = -op_cycles;
221 if (!unconditional) {
222 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
223 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
224 _jit, op->i.rt, REG_EXT);
226 /* Unload dead registers before evaluating the branch */
227 if (OPT_EARLY_UNLOAD)
228 lightrec_do_early_unload(state, block, offset);
230 if (op_flag_local_branch(op->flags) &&
231 (op_flag_no_ds(op->flags) || !next->opcode) &&
232 is_forward && !lightrec_has_dirty_regs(reg_cache))
233 no_indirection = true;
236 pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
240 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
242 if (!unconditional) {
243 /* Generate the branch opcode */
245 addr = jit_new_node_pww(code, NULL, rs, rt);
247 lightrec_free_regs(reg_cache);
248 regs_backup = lightrec_regcache_enter_branch(reg_cache);
251 if (op_flag_local_branch(op->flags)) {
252 /* Recompile the delay slot */
253 if (next && next->opcode && !op_flag_no_ds(op->flags))
254 lightrec_rec_opcode(state, block, offset + 1);
257 /* Update the $ra register */
258 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
259 jit_movi(link_reg, link);
260 lightrec_free_reg(reg_cache, link_reg);
263 /* Clean remaining registers */
264 lightrec_clean_regs(reg_cache, _jit);
266 target_offset = offset + 1 + (s16)op->i.imm
267 - !!op_flag_no_ds(op->flags);
268 pr_debug("Adding local branch to offset 0x%x\n",
270 branch = &state->local_branches[
271 state->nb_local_branches++];
273 branch->target = target_offset;
276 branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
278 branch->branch = jit_b();
280 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
283 if (!op_flag_local_branch(op->flags) || !is_forward) {
284 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
285 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
289 if (!unconditional) {
293 lightrec_regcache_leave_branch(reg_cache, regs_backup);
296 /* Update the $ra register */
297 link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
299 jit_movi(link_reg, (s32)link);
300 lightrec_free_reg(reg_cache, link_reg);
303 if (!op_flag_no_ds(op->flags) && next->opcode)
304 lightrec_rec_opcode(state, block, offset + 1);
308 static void rec_BNE(struct lightrec_cstate *state,
309 const struct block *block, u16 offset)
311 union code c = block->opcode_list[offset].c;
313 _jit_name(block->_jit, __func__);
316 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
318 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
321 static void rec_BEQ(struct lightrec_cstate *state,
322 const struct block *block, u16 offset)
324 union code c = block->opcode_list[offset].c;
326 _jit_name(block->_jit, __func__);
329 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
331 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
334 static void rec_BLEZ(struct lightrec_cstate *state,
335 const struct block *block, u16 offset)
337 union code c = block->opcode_list[offset].c;
339 _jit_name(block->_jit, __func__);
340 rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
343 static void rec_BGTZ(struct lightrec_cstate *state,
344 const struct block *block, u16 offset)
346 _jit_name(block->_jit, __func__);
347 rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
350 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
351 const struct block *block, u16 offset)
353 _jit_name(block->_jit, __func__);
354 rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
357 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
358 const struct block *block, u16 offset)
360 _jit_name(block->_jit, __func__);
361 rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
362 get_branch_pc(block, offset, 2), false, true);
365 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
366 const struct block *block, u16 offset)
368 union code c = block->opcode_list[offset].c;
370 _jit_name(block->_jit, __func__);
371 rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
374 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
375 const struct block *block, u16 offset)
377 const struct opcode *op = &block->opcode_list[offset];
378 _jit_name(block->_jit, __func__);
379 rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
380 get_branch_pc(block, offset, 2),
384 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
385 u16 offset, jit_code_t code, bool slti)
387 struct regcache *reg_cache = state->reg_cache;
388 union code c = block->opcode_list[offset].c;
389 jit_state_t *_jit = block->_jit;
390 u8 rs, rt, out_flags = REG_EXT;
393 out_flags |= REG_ZEXT;
395 jit_note(__FILE__, __LINE__);
396 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
397 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
399 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
401 lightrec_free_reg(reg_cache, rs);
402 lightrec_free_reg(reg_cache, rt);
405 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
406 u16 offset, jit_code_t code, bool out_ext)
408 struct regcache *reg_cache = state->reg_cache;
409 union code c = block->opcode_list[offset].c;
410 jit_state_t *_jit = block->_jit;
413 jit_note(__FILE__, __LINE__);
414 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
415 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
416 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
417 out_ext ? REG_EXT | REG_ZEXT : 0);
419 jit_new_node_www(code, rd, rs, rt);
421 lightrec_free_reg(reg_cache, rs);
422 lightrec_free_reg(reg_cache, rt);
423 lightrec_free_reg(reg_cache, rd);
426 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
427 u16 offset, jit_code_t code)
429 struct regcache *reg_cache = state->reg_cache;
430 union code c = block->opcode_list[offset].c;
431 jit_state_t *_jit = block->_jit;
432 u8 rd, rt, rs, temp, flags = 0;
434 jit_note(__FILE__, __LINE__);
435 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
437 if (code == jit_code_rshr)
439 else if (code == jit_code_rshr_u)
442 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
443 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
445 if (rs != rd && rt != rd) {
446 jit_andi(rd, rs, 0x1f);
447 jit_new_node_www(code, rd, rt, rd);
449 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
450 jit_andi(temp, rs, 0x1f);
451 jit_new_node_www(code, rd, rt, temp);
452 lightrec_free_reg(reg_cache, temp);
455 lightrec_free_reg(reg_cache, rs);
456 lightrec_free_reg(reg_cache, rt);
457 lightrec_free_reg(reg_cache, rd);
460 static void rec_movi(struct lightrec_cstate *state,
461 const struct block *block, u16 offset)
463 struct regcache *reg_cache = state->reg_cache;
464 union code c = block->opcode_list[offset].c;
465 jit_state_t *_jit = block->_jit;
469 if (!(c.i.imm & 0x8000))
472 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
474 jit_movi(rt, (s32)(s16) c.i.imm);
476 lightrec_free_reg(reg_cache, rt);
479 static void rec_ADDIU(struct lightrec_cstate *state,
480 const struct block *block, u16 offset)
482 _jit_name(block->_jit, __func__);
484 if (block->opcode_list[offset].c.i.rs)
485 rec_alu_imm(state, block, offset, jit_code_addi, false);
487 rec_movi(state, block, offset);
490 static void rec_ADDI(struct lightrec_cstate *state,
491 const struct block *block, u16 offset)
493 /* TODO: Handle the exception? */
494 _jit_name(block->_jit, __func__);
495 rec_ADDIU(state, block, offset);
498 static void rec_SLTIU(struct lightrec_cstate *state,
499 const struct block *block, u16 offset)
501 _jit_name(block->_jit, __func__);
502 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
505 static void rec_SLTI(struct lightrec_cstate *state,
506 const struct block *block, u16 offset)
508 _jit_name(block->_jit, __func__);
509 rec_alu_imm(state, block, offset, jit_code_lti, true);
512 static void rec_ANDI(struct lightrec_cstate *state,
513 const struct block *block, u16 offset)
515 struct regcache *reg_cache = state->reg_cache;
516 union code c = block->opcode_list[offset].c;
517 jit_state_t *_jit = block->_jit;
520 _jit_name(block->_jit, __func__);
521 jit_note(__FILE__, __LINE__);
522 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
523 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
526 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
527 * casts to uint8_t / uint16_t. */
530 else if (c.i.imm == 0xffff)
533 jit_andi(rt, rs, (u32)(u16) c.i.imm);
535 lightrec_free_reg(reg_cache, rs);
536 lightrec_free_reg(reg_cache, rt);
539 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
540 u16 offset, jit_code_t code)
542 struct regcache *reg_cache = state->reg_cache;
543 union code c = block->opcode_list[offset].c;
544 jit_state_t *_jit = block->_jit;
547 jit_note(__FILE__, __LINE__);
548 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
549 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
551 flags = lightrec_get_reg_in_flags(reg_cache, rs);
552 lightrec_set_reg_out_flags(reg_cache, rt, flags);
554 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
556 lightrec_free_reg(reg_cache, rs);
557 lightrec_free_reg(reg_cache, rt);
561 static void rec_ORI(struct lightrec_cstate *state,
562 const struct block *block, u16 offset)
564 _jit_name(block->_jit, __func__);
565 rec_alu_or_xor(state, block, offset, jit_code_ori);
568 static void rec_XORI(struct lightrec_cstate *state,
569 const struct block *block, u16 offset)
571 _jit_name(block->_jit, __func__);
572 rec_alu_or_xor(state, block, offset, jit_code_xori);
575 static void rec_LUI(struct lightrec_cstate *state,
576 const struct block *block, u16 offset)
578 struct regcache *reg_cache = state->reg_cache;
579 union code c = block->opcode_list[offset].c;
580 jit_state_t *_jit = block->_jit;
581 u8 rt, flags = REG_EXT;
584 jit_note(__FILE__, __LINE__);
586 if (!(c.i.imm & BIT(15)))
589 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
591 jit_movi(rt, (s32)(c.i.imm << 16));
593 lightrec_free_reg(reg_cache, rt);
596 static void rec_special_ADDU(struct lightrec_cstate *state,
597 const struct block *block, u16 offset)
599 _jit_name(block->_jit, __func__);
600 rec_alu_special(state, block, offset, jit_code_addr, false);
603 static void rec_special_ADD(struct lightrec_cstate *state,
604 const struct block *block, u16 offset)
606 /* TODO: Handle the exception? */
607 _jit_name(block->_jit, __func__);
608 rec_alu_special(state, block, offset, jit_code_addr, false);
611 static void rec_special_SUBU(struct lightrec_cstate *state,
612 const struct block *block, u16 offset)
614 _jit_name(block->_jit, __func__);
615 rec_alu_special(state, block, offset, jit_code_subr, false);
618 static void rec_special_SUB(struct lightrec_cstate *state,
619 const struct block *block, u16 offset)
621 /* TODO: Handle the exception? */
622 _jit_name(block->_jit, __func__);
623 rec_alu_special(state, block, offset, jit_code_subr, false);
626 static void rec_special_AND(struct lightrec_cstate *state,
627 const struct block *block, u16 offset)
629 struct regcache *reg_cache = state->reg_cache;
630 union code c = block->opcode_list[offset].c;
631 jit_state_t *_jit = block->_jit;
632 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
634 _jit_name(block->_jit, __func__);
635 jit_note(__FILE__, __LINE__);
636 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
637 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
638 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
640 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
641 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
643 /* Z(rd) = Z(rs) | Z(rt) */
644 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
646 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
647 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
648 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
649 (REG_EXT & flags_rs & flags_rt))
652 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
654 jit_andr(rd, rs, rt);
656 lightrec_free_reg(reg_cache, rs);
657 lightrec_free_reg(reg_cache, rt);
658 lightrec_free_reg(reg_cache, rd);
661 static void rec_special_or_nor(struct lightrec_cstate *state,
662 const struct block *block, u16 offset, bool nor)
664 struct regcache *reg_cache = state->reg_cache;
665 union code c = block->opcode_list[offset].c;
666 jit_state_t *_jit = block->_jit;
667 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
669 jit_note(__FILE__, __LINE__);
670 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
671 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
672 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
674 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
675 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
677 /* or: Z(rd) = Z(rs) & Z(rt)
680 flags_rd = REG_ZEXT & flags_rs & flags_rt;
682 /* E(rd) = E(rs) & E(rt) */
683 if (REG_EXT & flags_rs & flags_rt)
686 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
693 lightrec_free_reg(reg_cache, rs);
694 lightrec_free_reg(reg_cache, rt);
695 lightrec_free_reg(reg_cache, rd);
698 static void rec_special_OR(struct lightrec_cstate *state,
699 const struct block *block, u16 offset)
701 _jit_name(block->_jit, __func__);
702 rec_special_or_nor(state, block, offset, false);
705 static void rec_special_NOR(struct lightrec_cstate *state,
706 const struct block *block, u16 offset)
708 _jit_name(block->_jit, __func__);
709 rec_special_or_nor(state, block, offset, true);
712 static void rec_special_XOR(struct lightrec_cstate *state,
713 const struct block *block, u16 offset)
715 struct regcache *reg_cache = state->reg_cache;
716 union code c = block->opcode_list[offset].c;
717 jit_state_t *_jit = block->_jit;
718 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
720 _jit_name(block->_jit, __func__);
722 jit_note(__FILE__, __LINE__);
723 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
724 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
725 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
727 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
728 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
730 /* Z(rd) = Z(rs) & Z(rt) */
731 flags_rd = REG_ZEXT & flags_rs & flags_rt;
733 /* E(rd) = E(rs) & E(rt) */
734 flags_rd |= REG_EXT & flags_rs & flags_rt;
736 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
738 jit_xorr(rd, rs, rt);
740 lightrec_free_reg(reg_cache, rs);
741 lightrec_free_reg(reg_cache, rt);
742 lightrec_free_reg(reg_cache, rd);
745 static void rec_special_SLTU(struct lightrec_cstate *state,
746 const struct block *block, u16 offset)
748 _jit_name(block->_jit, __func__);
749 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
752 static void rec_special_SLT(struct lightrec_cstate *state,
753 const struct block *block, u16 offset)
755 _jit_name(block->_jit, __func__);
756 rec_alu_special(state, block, offset, jit_code_ltr, true);
759 static void rec_special_SLLV(struct lightrec_cstate *state,
760 const struct block *block, u16 offset)
762 _jit_name(block->_jit, __func__);
763 rec_alu_shiftv(state, block, offset, jit_code_lshr);
766 static void rec_special_SRLV(struct lightrec_cstate *state,
767 const struct block *block, u16 offset)
769 _jit_name(block->_jit, __func__);
770 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
773 static void rec_special_SRAV(struct lightrec_cstate *state,
774 const struct block *block, u16 offset)
776 _jit_name(block->_jit, __func__);
777 rec_alu_shiftv(state, block, offset, jit_code_rshr);
780 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
781 u16 offset, jit_code_t code)
783 struct regcache *reg_cache = state->reg_cache;
784 union code c = block->opcode_list[offset].c;
785 jit_state_t *_jit = block->_jit;
786 u8 rd, rt, flags = 0;
788 jit_note(__FILE__, __LINE__);
790 if (code == jit_code_rshi)
792 else if (code == jit_code_rshi_u)
795 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
797 /* Input reg is zero-extended, if we SRL at least by one bit, we know
798 * the output reg will be both zero-extended and sign-extended. */
799 if (code == jit_code_rshi_u && c.r.imm)
801 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
803 jit_new_node_www(code, rd, rt, c.r.imm);
805 lightrec_free_reg(reg_cache, rt);
806 lightrec_free_reg(reg_cache, rd);
809 static void rec_special_SLL(struct lightrec_cstate *state,
810 const struct block *block, u16 offset)
812 _jit_name(block->_jit, __func__);
813 rec_alu_shift(state, block, offset, jit_code_lshi);
816 static void rec_special_SRL(struct lightrec_cstate *state,
817 const struct block *block, u16 offset)
819 _jit_name(block->_jit, __func__);
820 rec_alu_shift(state, block, offset, jit_code_rshi_u);
823 static void rec_special_SRA(struct lightrec_cstate *state,
824 const struct block *block, u16 offset)
826 _jit_name(block->_jit, __func__);
827 rec_alu_shift(state, block, offset, jit_code_rshi);
830 static void rec_alu_mult(struct lightrec_cstate *state,
831 const struct block *block, u16 offset, bool is_signed)
833 struct regcache *reg_cache = state->reg_cache;
834 union code c = block->opcode_list[offset].c;
835 u32 flags = block->opcode_list[offset].flags;
836 u8 reg_lo = get_mult_div_lo(c);
837 u8 reg_hi = get_mult_div_hi(c);
838 jit_state_t *_jit = block->_jit;
839 u8 lo, hi, rs, rt, rflags = 0;
841 jit_note(__FILE__, __LINE__);
848 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
849 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
851 if (!op_flag_no_lo(flags))
852 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
853 else if (__WORDSIZE == 32)
854 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
856 if (!op_flag_no_hi(flags))
857 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
859 if (__WORDSIZE == 32) {
860 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
861 * operation if the MULT was detected a 32-bit only. */
862 if (!op_flag_no_hi(flags)) {
864 jit_qmulr(lo, hi, rs, rt);
866 jit_qmulr_u(lo, hi, rs, rt);
868 jit_mulr(lo, rs, rt);
871 /* On 64-bit systems, do a 64*64->64 bit operation. */
872 if (op_flag_no_lo(flags)) {
873 jit_mulr(hi, rs, rt);
874 jit_rshi(hi, hi, 32);
876 jit_mulr(lo, rs, rt);
878 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
879 if (!op_flag_no_hi(flags))
880 jit_rshi(hi, lo, 32);
884 lightrec_free_reg(reg_cache, rs);
885 lightrec_free_reg(reg_cache, rt);
886 if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
887 lightrec_free_reg(reg_cache, lo);
888 if (!op_flag_no_hi(flags))
889 lightrec_free_reg(reg_cache, hi);
892 static void rec_alu_div(struct lightrec_cstate *state,
893 const struct block *block, u16 offset, bool is_signed)
895 struct regcache *reg_cache = state->reg_cache;
896 union code c = block->opcode_list[offset].c;
897 u32 flags = block->opcode_list[offset].flags;
898 bool no_check = op_flag_no_div_check(flags);
899 u8 reg_lo = get_mult_div_lo(c);
900 u8 reg_hi = get_mult_div_hi(c);
901 jit_state_t *_jit = block->_jit;
902 jit_node_t *branch, *to_end;
903 u8 lo = 0, hi = 0, rs, rt, rflags = 0;
905 jit_note(__FILE__, __LINE__);
912 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
913 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
915 if (!op_flag_no_lo(flags))
916 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
918 if (!op_flag_no_hi(flags))
919 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
921 /* Jump to special handler if dividing by zero */
923 branch = jit_beqi(rt, 0);
925 if (op_flag_no_lo(flags)) {
927 jit_remr(hi, rs, rt);
929 jit_remr_u(hi, rs, rt);
930 } else if (op_flag_no_hi(flags)) {
932 jit_divr(lo, rs, rt);
934 jit_divr_u(lo, rs, rt);
937 jit_qdivr(lo, hi, rs, rt);
939 jit_qdivr_u(lo, hi, rs, rt);
943 /* Jump above the div-by-zero handler */
948 if (!op_flag_no_lo(flags)) {
954 jit_movi(lo, 0xffffffff);
958 if (!op_flag_no_hi(flags))
964 lightrec_free_reg(reg_cache, rs);
965 lightrec_free_reg(reg_cache, rt);
967 if (!op_flag_no_lo(flags))
968 lightrec_free_reg(reg_cache, lo);
970 if (!op_flag_no_hi(flags))
971 lightrec_free_reg(reg_cache, hi);
974 static void rec_special_MULT(struct lightrec_cstate *state,
975 const struct block *block, u16 offset)
977 _jit_name(block->_jit, __func__);
978 rec_alu_mult(state, block, offset, true);
981 static void rec_special_MULTU(struct lightrec_cstate *state,
982 const struct block *block, u16 offset)
984 _jit_name(block->_jit, __func__);
985 rec_alu_mult(state, block, offset, false);
988 static void rec_special_DIV(struct lightrec_cstate *state,
989 const struct block *block, u16 offset)
991 _jit_name(block->_jit, __func__);
992 rec_alu_div(state, block, offset, true);
995 static void rec_special_DIVU(struct lightrec_cstate *state,
996 const struct block *block, u16 offset)
998 _jit_name(block->_jit, __func__);
999 rec_alu_div(state, block, offset, false);
1002 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1003 const struct block *block, u8 dst, u8 src)
1005 struct regcache *reg_cache = state->reg_cache;
1006 jit_state_t *_jit = block->_jit;
1008 jit_note(__FILE__, __LINE__);
1009 src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1010 dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1012 jit_extr_i(dst, src);
1014 lightrec_free_reg(reg_cache, src);
1015 lightrec_free_reg(reg_cache, dst);
1018 static void rec_special_MFHI(struct lightrec_cstate *state,
1019 const struct block *block, u16 offset)
1021 union code c = block->opcode_list[offset].c;
1023 _jit_name(block->_jit, __func__);
1024 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1027 static void rec_special_MTHI(struct lightrec_cstate *state,
1028 const struct block *block, u16 offset)
1030 union code c = block->opcode_list[offset].c;
1032 _jit_name(block->_jit, __func__);
1033 rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1036 static void rec_special_MFLO(struct lightrec_cstate *state,
1037 const struct block *block, u16 offset)
1039 union code c = block->opcode_list[offset].c;
1041 _jit_name(block->_jit, __func__);
1042 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1045 static void rec_special_MTLO(struct lightrec_cstate *state,
1046 const struct block *block, u16 offset)
1048 union code c = block->opcode_list[offset].c;
1050 _jit_name(block->_jit, __func__);
1051 rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1054 static void call_to_c_wrapper(struct lightrec_cstate *state,
1055 const struct block *block, u32 arg,
1056 enum c_wrappers wrapper)
1058 struct regcache *reg_cache = state->reg_cache;
1059 jit_state_t *_jit = block->_jit;
1062 /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1063 tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1065 tmp = lightrec_get_reg_with_value(reg_cache,
1066 (intptr_t) state->state->wrappers_eps[wrapper]);
1068 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1069 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1070 offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1072 lightrec_temp_set_value(reg_cache, tmp,
1073 (intptr_t) state->state->wrappers_eps[wrapper]);
1076 lightrec_free_reg(reg_cache, tmp2);
1079 /* On MIPS, register t9 is always used as the target register for JALR.
1080 * Therefore if it does not contain the target address we must
1083 lightrec_unload_reg(reg_cache, _jit, _T9);
1089 lightrec_regcache_mark_live(reg_cache, _jit);
1092 lightrec_free_reg(reg_cache, tmp);
1093 lightrec_regcache_mark_live(reg_cache, _jit);
1096 static void rec_io(struct lightrec_cstate *state,
1097 const struct block *block, u16 offset,
1098 bool load_rt, bool read_rt)
1100 struct regcache *reg_cache = state->reg_cache;
1101 jit_state_t *_jit = block->_jit;
1102 union code c = block->opcode_list[offset].c;
1103 u32 flags = block->opcode_list[offset].flags;
1104 bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1107 jit_note(__FILE__, __LINE__);
1109 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1111 if (read_rt && likely(c.i.rt))
1112 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1114 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1117 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1119 lut_entry = lightrec_get_lut_entry(block);
1120 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1121 C_WRAPPER_RW_GENERIC);
1125 static u32 rec_ram_mask(struct lightrec_state *state)
1127 return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1130 static void rec_store_memory(struct lightrec_cstate *cstate,
1131 const struct block *block,
1132 u16 offset, jit_code_t code,
1133 jit_code_t swap_code,
1134 uintptr_t addr_offset, u32 addr_mask,
1137 const struct lightrec_state *state = cstate->state;
1138 struct regcache *reg_cache = cstate->reg_cache;
1139 struct opcode *op = &block->opcode_list[offset];
1140 jit_state_t *_jit = block->_jit;
1141 union code c = op->c;
1142 u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1143 s16 imm = (s16)c.i.imm;
1144 s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1145 s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1146 bool no_mask = op_flag_no_mask(op->flags);
1147 bool add_imm = c.i.imm &&
1148 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1149 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1150 bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1152 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1153 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1155 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1160 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1168 jit_andi(tmp, addr_reg, addr_mask);
1173 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1174 jit_addi(tmp2, addr_reg, addr_offset);
1177 addr_reg2 = addr_reg;
1180 if (is_big_endian() && swap_code && c.i.rt) {
1181 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1183 jit_new_node_ww(swap_code, tmp3, rt);
1184 jit_new_node_www(code, imm, addr_reg2, tmp3);
1186 lightrec_free_reg(reg_cache, tmp3);
1188 jit_new_node_www(code, imm, addr_reg2, rt);
1191 lightrec_free_reg(reg_cache, rt);
1194 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1196 if (c.i.op != OP_SW) {
1197 jit_andi(tmp, addr_reg, ~3);
1201 if (!lut_is_32bit(state)) {
1202 jit_lshi(tmp, addr_reg, 1);
1206 if (addr_reg == rs && c.i.rs == 0) {
1207 addr_reg = LIGHTREC_REG_STATE;
1209 jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1213 if (lut_is_32bit(state))
1214 jit_stxi_i(lut_offt, addr_reg, tmp3);
1216 jit_stxi(lut_offt, addr_reg, tmp3);
1218 lightrec_free_reg(reg_cache, tmp3);
1222 lightrec_free_reg(reg_cache, tmp2);
1224 lightrec_free_reg(reg_cache, tmp);
1225 lightrec_free_reg(reg_cache, rs);
1228 static void rec_store_ram(struct lightrec_cstate *cstate,
1229 const struct block *block,
1230 u16 offset, jit_code_t code,
1231 jit_code_t swap_code, bool invalidate)
1233 struct lightrec_state *state = cstate->state;
1235 _jit_note(block->_jit, __FILE__, __LINE__);
1237 return rec_store_memory(cstate, block, offset, code, swap_code,
1238 state->offset_ram, rec_ram_mask(state),
1242 static void rec_store_scratch(struct lightrec_cstate *cstate,
1243 const struct block *block, u16 offset,
1244 jit_code_t code, jit_code_t swap_code)
1246 _jit_note(block->_jit, __FILE__, __LINE__);
1248 return rec_store_memory(cstate, block, offset, code, swap_code,
1249 cstate->state->offset_scratch,
1253 static void rec_store_io(struct lightrec_cstate *cstate,
1254 const struct block *block, u16 offset,
1255 jit_code_t code, jit_code_t swap_code)
1257 _jit_note(block->_jit, __FILE__, __LINE__);
1259 return rec_store_memory(cstate, block, offset, code, swap_code,
1260 cstate->state->offset_io,
1264 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1265 const struct block *block,
1266 u16 offset, jit_code_t code,
1267 jit_code_t swap_code)
1269 struct lightrec_state *state = cstate->state;
1270 struct regcache *reg_cache = cstate->reg_cache;
1271 union code c = block->opcode_list[offset].c;
1272 jit_state_t *_jit = block->_jit;
1273 jit_node_t *to_not_ram, *to_end;
1274 u8 tmp, tmp2, rs, rt;
1277 jit_note(__FILE__, __LINE__);
1278 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1279 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1281 if (state->offset_ram || state->offset_scratch)
1282 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1284 /* Convert to KUNSEG and avoid RAM mirrors */
1285 if (state->mirrors_mapped) {
1287 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1288 } else if (c.i.imm) {
1290 jit_addi(tmp, rs, (s16)c.i.imm);
1291 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1294 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1297 lightrec_free_reg(reg_cache, rs);
1299 if (state->offset_ram != state->offset_scratch) {
1300 to_not_ram = jit_bmsi(tmp, BIT(28));
1302 jit_movi(tmp2, state->offset_ram);
1305 jit_patch(to_not_ram);
1307 jit_movi(tmp2, state->offset_scratch);
1309 } else if (state->offset_ram) {
1310 jit_movi(tmp2, state->offset_ram);
1313 if (state->offset_ram || state->offset_scratch) {
1314 jit_addr(tmp, tmp, tmp2);
1315 lightrec_free_reg(reg_cache, tmp2);
1318 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1320 if (is_big_endian() && swap_code && c.i.rt) {
1321 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1323 jit_new_node_ww(swap_code, tmp2, rt);
1324 jit_new_node_www(code, imm, tmp, tmp2);
1326 lightrec_free_reg(reg_cache, tmp2);
1328 jit_new_node_www(code, imm, tmp, rt);
1331 lightrec_free_reg(reg_cache, rt);
1332 lightrec_free_reg(reg_cache, tmp);
1335 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1336 u16 offset, jit_code_t code, jit_code_t swap_code)
1338 struct lightrec_state *state = cstate->state;
1339 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1340 struct regcache *reg_cache = cstate->reg_cache;
1341 union code c = block->opcode_list[offset].c;
1342 jit_state_t *_jit = block->_jit;
1343 jit_node_t *to_not_ram, *to_end;
1344 u8 tmp, tmp2, tmp3, rs, rt;
1346 jit_note(__FILE__, __LINE__);
1348 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1349 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1350 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1352 /* Convert to KUNSEG and avoid RAM mirrors */
1354 jit_addi(tmp2, rs, (s16)c.i.imm);
1355 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1357 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1360 lightrec_free_reg(reg_cache, rs);
1361 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1363 to_not_ram = jit_bgti(tmp2, ram_size);
1365 /* Compute the offset to the code LUT */
1366 jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1367 if (!lut_is_32bit(state))
1368 jit_lshi(tmp, tmp, 1);
1369 jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1371 /* Write NULL to the code LUT to invalidate any block that's there */
1372 if (lut_is_32bit(state))
1373 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1375 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1377 if (state->offset_ram != state->offset_scratch) {
1378 jit_movi(tmp, state->offset_ram);
1383 jit_patch(to_not_ram);
1385 if (state->offset_ram || state->offset_scratch)
1386 jit_movi(tmp, state->offset_scratch);
1388 if (state->offset_ram != state->offset_scratch)
1391 if (state->offset_ram || state->offset_scratch)
1392 jit_addr(tmp2, tmp2, tmp);
1394 lightrec_free_reg(reg_cache, tmp);
1395 lightrec_free_reg(reg_cache, tmp3);
1397 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1399 if (is_big_endian() && swap_code && c.i.rt) {
1400 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1402 jit_new_node_ww(swap_code, tmp, rt);
1403 jit_new_node_www(code, 0, tmp2, tmp);
1405 lightrec_free_reg(reg_cache, tmp);
1407 jit_new_node_www(code, 0, tmp2, rt);
1410 lightrec_free_reg(reg_cache, rt);
1411 lightrec_free_reg(reg_cache, tmp2);
1414 static void rec_store(struct lightrec_cstate *state,
1415 const struct block *block, u16 offset,
1416 jit_code_t code, jit_code_t swap_code)
1418 u32 flags = block->opcode_list[offset].flags;
1419 bool no_invalidate = op_flag_no_invalidate(flags) ||
1420 state->state->invalidate_from_dma_only;
1422 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1423 case LIGHTREC_IO_RAM:
1424 rec_store_ram(state, block, offset, code,
1425 swap_code, !no_invalidate);
1427 case LIGHTREC_IO_SCRATCH:
1428 rec_store_scratch(state, block, offset, code, swap_code);
1430 case LIGHTREC_IO_DIRECT:
1431 if (no_invalidate) {
1432 rec_store_direct_no_invalidate(state, block, offset,
1435 rec_store_direct(state, block, offset, code, swap_code);
1438 case LIGHTREC_IO_DIRECT_HW:
1439 rec_store_io(state, block, offset, code, swap_code);
1442 rec_io(state, block, offset, true, false);
1447 static void rec_SB(struct lightrec_cstate *state,
1448 const struct block *block, u16 offset)
1450 _jit_name(block->_jit, __func__);
1451 rec_store(state, block, offset, jit_code_stxi_c, 0);
1454 static void rec_SH(struct lightrec_cstate *state,
1455 const struct block *block, u16 offset)
1457 _jit_name(block->_jit, __func__);
1458 rec_store(state, block, offset,
1459 jit_code_stxi_s, jit_code_bswapr_us);
1462 static void rec_SW(struct lightrec_cstate *state,
1463 const struct block *block, u16 offset)
1466 _jit_name(block->_jit, __func__);
1467 rec_store(state, block, offset,
1468 jit_code_stxi_i, jit_code_bswapr_ui);
1471 static void rec_SWL(struct lightrec_cstate *state,
1472 const struct block *block, u16 offset)
1474 _jit_name(block->_jit, __func__);
1475 rec_io(state, block, offset, true, false);
1478 static void rec_SWR(struct lightrec_cstate *state,
1479 const struct block *block, u16 offset)
1481 _jit_name(block->_jit, __func__);
1482 rec_io(state, block, offset, true, false);
1485 static void rec_SWC2(struct lightrec_cstate *state,
1486 const struct block *block, u16 offset)
1488 _jit_name(block->_jit, __func__);
1489 rec_io(state, block, offset, false, false);
1492 static void rec_load_memory(struct lightrec_cstate *cstate,
1493 const struct block *block, u16 offset,
1494 jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1495 uintptr_t addr_offset, u32 addr_mask)
1497 struct regcache *reg_cache = cstate->reg_cache;
1498 struct opcode *op = &block->opcode_list[offset];
1499 jit_state_t *_jit = block->_jit;
1500 u8 rs, rt, addr_reg, flags = REG_EXT;
1501 bool no_mask = op_flag_no_mask(op->flags);
1502 union code c = op->c;
1511 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1512 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1514 if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1515 jit_addi(rt, rs, (s16)c.i.imm);
1524 jit_andi(rt, addr_reg, addr_mask);
1529 jit_addi(rt, addr_reg, addr_offset);
1533 jit_new_node_www(code, rt, addr_reg, imm);
1535 if (is_big_endian() && swap_code) {
1536 jit_new_node_ww(swap_code, rt, rt);
1538 if (c.i.op == OP_LH)
1540 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1544 lightrec_free_reg(reg_cache, rs);
1545 lightrec_free_reg(reg_cache, rt);
1548 static void rec_load_ram(struct lightrec_cstate *cstate,
1549 const struct block *block, u16 offset,
1550 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1552 _jit_note(block->_jit, __FILE__, __LINE__);
1554 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1555 cstate->state->offset_ram, rec_ram_mask(cstate->state));
1558 static void rec_load_bios(struct lightrec_cstate *cstate,
1559 const struct block *block, u16 offset,
1560 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1562 _jit_note(block->_jit, __FILE__, __LINE__);
1564 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1565 cstate->state->offset_bios, 0x1fffffff);
1568 static void rec_load_scratch(struct lightrec_cstate *cstate,
1569 const struct block *block, u16 offset,
1570 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1572 _jit_note(block->_jit, __FILE__, __LINE__);
1574 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1575 cstate->state->offset_scratch, 0x1fffffff);
1578 static void rec_load_io(struct lightrec_cstate *cstate,
1579 const struct block *block, u16 offset,
1580 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1582 _jit_note(block->_jit, __FILE__, __LINE__);
1584 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1585 cstate->state->offset_io, 0x1fffffff);
1588 static void rec_load_direct(struct lightrec_cstate *cstate,
1589 const struct block *block, u16 offset,
1590 jit_code_t code, jit_code_t swap_code,
1593 struct lightrec_state *state = cstate->state;
1594 struct regcache *reg_cache = cstate->reg_cache;
1595 union code c = block->opcode_list[offset].c;
1596 jit_state_t *_jit = block->_jit;
1597 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1598 u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1607 jit_note(__FILE__, __LINE__);
1608 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1609 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1611 if ((state->offset_ram == state->offset_bios &&
1612 state->offset_ram == state->offset_scratch &&
1613 state->mirrors_mapped) || !c.i.imm) {
1617 jit_addi(rt, rs, (s16)c.i.imm);
1621 if (c.i.rs != c.i.rt)
1622 lightrec_free_reg(reg_cache, rs);
1625 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1627 if (state->offset_ram == state->offset_bios &&
1628 state->offset_ram == state->offset_scratch) {
1629 if (!state->mirrors_mapped) {
1630 jit_andi(tmp, addr_reg, BIT(28));
1631 jit_rshi_u(tmp, tmp, 28 - 22);
1632 jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1633 jit_andr(rt, addr_reg, tmp);
1635 jit_andi(rt, addr_reg, 0x1fffffff);
1638 if (state->offset_ram)
1639 jit_movi(tmp, state->offset_ram);
1641 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1643 /* Convert to KUNSEG and avoid RAM mirrors */
1644 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1646 if (state->offset_ram)
1647 jit_movi(tmp, state->offset_ram);
1651 jit_patch(to_not_ram);
1653 if (state->offset_bios != state->offset_scratch)
1654 to_not_bios = jit_bmci(addr_reg, BIT(22));
1656 /* Convert to KUNSEG */
1657 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1659 jit_movi(tmp, state->offset_bios);
1661 if (state->offset_bios != state->offset_scratch) {
1664 jit_patch(to_not_bios);
1666 /* Convert to KUNSEG */
1667 jit_andi(rt, addr_reg, 0x1f800fff);
1669 if (state->offset_scratch)
1670 jit_movi(tmp, state->offset_scratch);
1678 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1679 jit_addr(rt, rt, tmp);
1681 jit_new_node_www(code, rt, rt, imm);
1683 if (is_big_endian() && swap_code) {
1684 jit_new_node_ww(swap_code, rt, rt);
1686 if (c.i.op == OP_LH)
1688 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1692 lightrec_free_reg(reg_cache, addr_reg);
1693 lightrec_free_reg(reg_cache, rt);
1694 lightrec_free_reg(reg_cache, tmp);
1697 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1698 u16 offset, jit_code_t code, jit_code_t swap_code,
1701 u32 flags = block->opcode_list[offset].flags;
1703 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1704 case LIGHTREC_IO_RAM:
1705 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1707 case LIGHTREC_IO_BIOS:
1708 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1710 case LIGHTREC_IO_SCRATCH:
1711 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1713 case LIGHTREC_IO_DIRECT_HW:
1714 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1716 case LIGHTREC_IO_DIRECT:
1717 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1720 rec_io(state, block, offset, false, true);
1725 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1727 _jit_name(block->_jit, __func__);
1728 rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1731 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1733 _jit_name(block->_jit, __func__);
1734 rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1737 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1739 jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1741 _jit_name(block->_jit, __func__);
1742 rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1745 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1747 _jit_name(block->_jit, __func__);
1748 rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1751 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1753 _jit_name(block->_jit, __func__);
1754 rec_io(state, block, offset, true, true);
1757 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1759 _jit_name(block->_jit, __func__);
1760 rec_io(state, block, offset, true, true);
1763 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1767 if (is_big_endian() && __WORDSIZE == 64)
1768 code = jit_code_ldxi_ui;
1770 code = jit_code_ldxi_i;
1772 _jit_name(block->_jit, __func__);
1773 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1776 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1778 _jit_name(block->_jit, __func__);
1779 rec_io(state, block, offset, false, false);
1782 static void rec_break_syscall(struct lightrec_cstate *state,
1783 const struct block *block, u16 offset,
1786 struct regcache *reg_cache = state->reg_cache;
1787 jit_state_t *_jit = block->_jit;
1790 _jit_note(block->_jit, __FILE__, __LINE__);
1792 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1794 jit_movi(tmp, exit_code);
1795 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1796 LIGHTREC_REG_STATE, tmp);
1798 lightrec_free_reg(reg_cache, tmp);
1800 /* TODO: the return address should be "pc - 4" if we're a delay slot */
1801 lightrec_emit_end_of_block(state, block, offset, -1,
1802 get_ds_pc(block, offset, 0),
1806 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1807 const struct block *block, u16 offset)
1809 _jit_name(block->_jit, __func__);
1810 rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1813 static void rec_special_BREAK(struct lightrec_cstate *state,
1814 const struct block *block, u16 offset)
1816 _jit_name(block->_jit, __func__);
1817 rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1820 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1822 struct regcache *reg_cache = state->reg_cache;
1823 union code c = block->opcode_list[offset].c;
1824 jit_state_t *_jit = block->_jit;
1826 jit_note(__FILE__, __LINE__);
1827 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1829 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1832 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1834 struct regcache *reg_cache = state->reg_cache;
1835 union code c = block->opcode_list[offset].c;
1836 jit_state_t *_jit = block->_jit;
1838 jit_note(__FILE__, __LINE__);
1839 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1840 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1842 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1844 if (c.i.op == OP_CP0 &&
1845 !op_flag_no_ds(block->opcode_list[offset].flags) &&
1846 (c.r.rd == 12 || c.r.rd == 13))
1847 lightrec_emit_end_of_block(state, block, offset, -1,
1848 get_ds_pc(block, offset, 1),
1853 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1855 struct regcache *reg_cache = state->reg_cache;
1856 union code c = block->opcode_list[offset].c;
1857 jit_state_t *_jit = block->_jit;
1860 jit_note(__FILE__, __LINE__);
1862 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1864 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1865 offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1867 lightrec_free_reg(reg_cache, rt);
1870 static bool block_in_bios(const struct lightrec_cstate *state,
1871 const struct block *block)
1873 const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1874 u32 pc = kunseg(block->pc);
1876 return pc >= bios->pc && pc < bios->pc + bios->length;
1880 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1882 struct regcache *reg_cache = state->reg_cache;
1883 const union code c = block->opcode_list[offset].c;
1884 jit_state_t *_jit = block->_jit;
1885 u8 rt, tmp = 0, tmp2, status;
1887 jit_note(__FILE__, __LINE__);
1895 /* Those registers are read-only */
1901 if (block_in_bios(state, block) && c.r.rd == 12) {
1902 /* If we are running code from the BIOS, handle writes to the
1903 * Status register in C. BIOS code may toggle bit 16 which will
1904 * map/unmap the RAM, while game code cannot do that. */
1905 rec_mtc(state, block, offset);
1909 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1912 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1913 LIGHTREC_REG_STATE, rt);
1916 if (c.r.rd == 12 || c.r.rd == 13) {
1917 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1918 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1919 offsetof(struct lightrec_state, regs.cp0[13]));
1921 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1926 } else if (c.r.rd == 13) {
1927 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1928 jit_andi(tmp2, rt, 0x0300);
1929 jit_ori(tmp, tmp, 0x0300);
1930 jit_xori(tmp, tmp, 0x0300);
1931 jit_orr(tmp, tmp, tmp2);
1932 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1933 offsetof(struct lightrec_state, regs.cp0[12]));
1934 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1935 LIGHTREC_REG_STATE, tmp);
1939 if (c.r.rd == 12 || c.r.rd == 13) {
1940 /* Exit dynarec in case there's a software interrupt.
1941 * exit_flags = !!(status & tmp & 0x0300) & status; */
1942 jit_andr(tmp, tmp, status);
1943 jit_andi(tmp, tmp, 0x0300);
1944 jit_nei(tmp, tmp, 0);
1945 jit_andr(tmp, tmp, status);
1949 /* Exit dynarec in case we unmask a hardware interrupt.
1950 * exit_flags = !(~status & 0x401) */
1952 jit_comr(tmp2, status);
1953 jit_andi(tmp2, tmp2, 0x401);
1954 jit_eqi(tmp2, tmp2, 0);
1955 jit_orr(tmp, tmp, tmp2);
1958 if (c.r.rd == 12 || c.r.rd == 13) {
1959 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1960 LIGHTREC_REG_STATE, tmp);
1962 lightrec_free_reg(reg_cache, tmp);
1963 lightrec_free_reg(reg_cache, tmp2);
1966 lightrec_free_reg(reg_cache, rt);
1968 if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1969 (c.r.rd == 12 || c.r.rd == 13))
1970 lightrec_emit_eob(state, block, offset + 1, true);
1973 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1974 const struct block *block, u16 offset)
1976 _jit_name(block->_jit, __func__);
1977 rec_mfc0(state, block, offset);
1980 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1981 const struct block *block, u16 offset)
1983 _jit_name(block->_jit, __func__);
1984 rec_mfc0(state, block, offset);
1987 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1988 const struct block *block, u16 offset)
1990 _jit_name(block->_jit, __func__);
1991 rec_mtc0(state, block, offset);
1994 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1995 const struct block *block, u16 offset)
1997 _jit_name(block->_jit, __func__);
1998 rec_mtc0(state, block, offset);
2001 static unsigned int cp2d_i_offset(u8 reg)
2003 return offsetof(struct lightrec_state, regs.cp2d[reg]);
2006 static unsigned int cp2d_s_offset(u8 reg)
2008 return cp2d_i_offset(reg) + is_big_endian() * 2;
2011 static unsigned int cp2c_i_offset(u8 reg)
2013 return offsetof(struct lightrec_state, regs.cp2c[reg]);
2016 static unsigned int cp2c_s_offset(u8 reg)
2018 return cp2c_i_offset(reg) + is_big_endian() * 2;
2021 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2022 const struct block *block, u16 offset)
2024 struct regcache *reg_cache = state->reg_cache;
2025 const union code c = block->opcode_list[offset].c;
2026 jit_state_t *_jit = block->_jit;
2027 const u32 zext_regs = 0x300f0080;
2028 u8 rt, tmp, tmp2, tmp3, out, flags;
2029 u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2032 _jit_name(block->_jit, __func__);
2034 if (state->state->ops.cop2_notify) {
2035 /* We must call cop2_notify, handle that in C. */
2036 rec_mfc(state, block, offset);
2040 flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2041 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2051 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2058 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2062 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2063 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2064 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2066 for (i = 0; i < 3; i++) {
2067 out = i == 0 ? rt : tmp;
2069 jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2070 jit_movi(tmp2, 0x1f);
2071 jit_rshi(out, tmp, 7);
2073 jit_ltr(tmp3, tmp2, out);
2074 jit_movnr(out, tmp2, tmp3);
2076 jit_gei(tmp2, out, 0);
2077 jit_movzr(out, tmp2, tmp2);
2080 jit_lshi(tmp, tmp, 5 * i);
2081 jit_orr(rt, rt, tmp);
2086 lightrec_free_reg(reg_cache, tmp);
2087 lightrec_free_reg(reg_cache, tmp2);
2088 lightrec_free_reg(reg_cache, tmp3);
2091 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2095 lightrec_free_reg(reg_cache, rt);
2098 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2099 const struct block *block, u16 offset)
2101 struct regcache *reg_cache = state->reg_cache;
2102 const union code c = block->opcode_list[offset].c;
2103 jit_state_t *_jit = block->_jit;
2106 _jit_name(block->_jit, __func__);
2108 if (state->state->ops.cop2_notify) {
2109 /* We must call cop2_notify, handle that in C. */
2110 rec_mfc(state, block, offset);
2122 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2123 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2126 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2127 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2131 lightrec_free_reg(reg_cache, rt);
2134 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2135 const struct block *block, u16 offset)
2137 struct regcache *reg_cache = state->reg_cache;
2138 const union code c = block->opcode_list[offset].c;
2139 jit_state_t *_jit = block->_jit;
2140 jit_node_t *loop, *to_loop;
2141 u8 rt, tmp, tmp2, flags = 0;
2143 _jit_name(block->_jit, __func__);
2145 if (state->state->ops.cop2_notify) {
2146 /* We must call cop2_notify, handle that in C. */
2147 rec_mtc(state, block, offset);
2157 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2161 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2162 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2164 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2165 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2167 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2168 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2169 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2171 lightrec_free_reg(reg_cache, tmp);
2172 lightrec_free_reg(reg_cache, tmp2);
2175 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2177 jit_lshi(tmp, rt, 7);
2178 jit_andi(tmp, tmp, 0xf80);
2179 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2181 jit_lshi(tmp, rt, 2);
2182 jit_andi(tmp, tmp, 0xf80);
2183 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2185 jit_rshi(tmp, rt, 3);
2186 jit_andi(tmp, tmp, 0xf80);
2187 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2189 lightrec_free_reg(reg_cache, tmp);
2192 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2193 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2195 /* if (rt < 0) rt = ~rt; */
2196 jit_rshi(tmp, rt, 31);
2197 jit_xorr(tmp, rt, tmp);
2199 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2200 jit_lshi(tmp, tmp, 1);
2203 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2205 jit_subi(tmp2, tmp2, 1);
2206 jit_rshi_u(tmp, tmp, 1);
2207 to_loop = jit_bnei(tmp, 0);
2209 jit_patch_at(to_loop, loop);
2211 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2212 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2214 lightrec_free_reg(reg_cache, tmp);
2215 lightrec_free_reg(reg_cache, tmp2);
2218 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2222 lightrec_free_reg(reg_cache, rt);
2225 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2226 const struct block *block, u16 offset)
2228 struct regcache *reg_cache = state->reg_cache;
2229 const union code c = block->opcode_list[offset].c;
2230 jit_state_t *_jit = block->_jit;
2233 _jit_name(block->_jit, __func__);
2235 if (state->state->ops.cop2_notify) {
2236 /* We must call cop2_notify, handle that in C. */
2237 rec_mtc(state, block, offset);
2241 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2251 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2254 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2255 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2257 jit_andi(tmp, rt, 0x7f87e000);
2258 jit_nei(tmp, tmp, 0);
2259 jit_lshi(tmp, tmp, 31);
2261 jit_andi(tmp2, rt, 0x7ffff000);
2262 jit_orr(tmp, tmp2, tmp);
2264 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2266 lightrec_free_reg(reg_cache, tmp);
2267 lightrec_free_reg(reg_cache, tmp2);
2271 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2274 lightrec_free_reg(reg_cache, rt);
2277 static void rec_cp0_RFE(struct lightrec_cstate *state,
2278 const struct block *block, u16 offset)
2280 struct regcache *reg_cache = state->reg_cache;
2281 jit_state_t *_jit = block->_jit;
2285 jit_note(__FILE__, __LINE__);
2287 status = lightrec_alloc_reg_temp(reg_cache, _jit);
2288 jit_ldxi_i(status, LIGHTREC_REG_STATE,
2289 offsetof(struct lightrec_state, regs.cp0[12]));
2291 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2293 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2294 jit_rshi(tmp, status, 2);
2295 jit_andi(tmp, tmp, 0xf);
2296 jit_andi(status, status, ~0xful);
2297 jit_orr(status, status, tmp);
2299 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2300 offsetof(struct lightrec_state, regs.cp0[13]));
2301 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2302 LIGHTREC_REG_STATE, status);
2304 /* Exit dynarec in case there's a software interrupt.
2305 * exit_flags = !!(status & cause & 0x0300) & status; */
2306 jit_andr(tmp, tmp, status);
2307 jit_andi(tmp, tmp, 0x0300);
2308 jit_nei(tmp, tmp, 0);
2309 jit_andr(tmp, tmp, status);
2310 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2311 LIGHTREC_REG_STATE, tmp);
2313 lightrec_free_reg(reg_cache, status);
2314 lightrec_free_reg(reg_cache, tmp);
2317 static void rec_CP(struct lightrec_cstate *state,
2318 const struct block *block, u16 offset)
2320 union code c = block->opcode_list[offset].c;
2321 jit_state_t *_jit = block->_jit;
2324 jit_note(__FILE__, __LINE__);
2326 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2329 static void rec_meta_MOV(struct lightrec_cstate *state,
2330 const struct block *block, u16 offset)
2332 struct regcache *reg_cache = state->reg_cache;
2333 union code c = block->opcode_list[offset].c;
2334 jit_state_t *_jit = block->_jit;
2337 _jit_name(block->_jit, __func__);
2338 jit_note(__FILE__, __LINE__);
2340 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2341 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2349 lightrec_free_reg(reg_cache, rs);
2350 lightrec_free_reg(reg_cache, rd);
2353 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2354 const struct block *block,
2357 struct regcache *reg_cache = state->reg_cache;
2358 union code c = block->opcode_list[offset].c;
2359 jit_state_t *_jit = block->_jit;
2362 _jit_name(block->_jit, __func__);
2363 jit_note(__FILE__, __LINE__);
2365 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2366 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2368 if (c.i.op == OP_META_EXTC)
2373 lightrec_free_reg(reg_cache, rs);
2374 lightrec_free_reg(reg_cache, rt);
2377 static void rec_meta_MULT2(struct lightrec_cstate *state,
2378 const struct block *block,
2381 struct regcache *reg_cache = state->reg_cache;
2382 union code c = block->opcode_list[offset].c;
2383 jit_state_t *_jit = block->_jit;
2384 u8 reg_lo = get_mult_div_lo(c);
2385 u8 reg_hi = get_mult_div_hi(c);
2386 u32 flags = block->opcode_list[offset].flags;
2387 bool is_signed = c.i.op == OP_META_MULT2;
2388 u8 rs, lo, hi, rflags = 0, hiflags = 0;
2390 if (!op_flag_no_hi(flags) && c.r.op < 32) {
2391 rflags = is_signed ? REG_EXT : REG_ZEXT;
2392 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2395 _jit_name(block->_jit, __func__);
2396 jit_note(__FILE__, __LINE__);
2398 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2400 if (!op_flag_no_lo(flags)) {
2401 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2404 jit_lshi(lo, rs, c.r.op);
2408 lightrec_free_reg(reg_cache, lo);
2411 if (!op_flag_no_hi(flags)) {
2412 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags);
2415 jit_lshi(hi, rs, c.r.op - 32);
2417 jit_rshi(hi, rs, 32 - c.r.op);
2419 jit_rshi_u(hi, rs, 32 - c.r.op);
2421 lightrec_free_reg(reg_cache, hi);
2424 lightrec_free_reg(reg_cache, rs);
2426 _jit_name(block->_jit, __func__);
2427 jit_note(__FILE__, __LINE__);
2430 static const lightrec_rec_func_t rec_standard[64] = {
2431 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2432 [OP_SPECIAL] = rec_SPECIAL,
2433 [OP_REGIMM] = rec_REGIMM,
2438 [OP_BLEZ] = rec_BLEZ,
2439 [OP_BGTZ] = rec_BGTZ,
2440 [OP_ADDI] = rec_ADDI,
2441 [OP_ADDIU] = rec_ADDIU,
2442 [OP_SLTI] = rec_SLTI,
2443 [OP_SLTIU] = rec_SLTIU,
2444 [OP_ANDI] = rec_ANDI,
2446 [OP_XORI] = rec_XORI,
2462 [OP_LWC2] = rec_LWC2,
2463 [OP_SWC2] = rec_SWC2,
2465 [OP_META_MOV] = rec_meta_MOV,
2466 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
2467 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
2468 [OP_META_MULT2] = rec_meta_MULT2,
2469 [OP_META_MULTU2] = rec_meta_MULT2,
2472 static const lightrec_rec_func_t rec_special[64] = {
2473 SET_DEFAULT_ELM(rec_special, unknown_opcode),
2474 [OP_SPECIAL_SLL] = rec_special_SLL,
2475 [OP_SPECIAL_SRL] = rec_special_SRL,
2476 [OP_SPECIAL_SRA] = rec_special_SRA,
2477 [OP_SPECIAL_SLLV] = rec_special_SLLV,
2478 [OP_SPECIAL_SRLV] = rec_special_SRLV,
2479 [OP_SPECIAL_SRAV] = rec_special_SRAV,
2480 [OP_SPECIAL_JR] = rec_special_JR,
2481 [OP_SPECIAL_JALR] = rec_special_JALR,
2482 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
2483 [OP_SPECIAL_BREAK] = rec_special_BREAK,
2484 [OP_SPECIAL_MFHI] = rec_special_MFHI,
2485 [OP_SPECIAL_MTHI] = rec_special_MTHI,
2486 [OP_SPECIAL_MFLO] = rec_special_MFLO,
2487 [OP_SPECIAL_MTLO] = rec_special_MTLO,
2488 [OP_SPECIAL_MULT] = rec_special_MULT,
2489 [OP_SPECIAL_MULTU] = rec_special_MULTU,
2490 [OP_SPECIAL_DIV] = rec_special_DIV,
2491 [OP_SPECIAL_DIVU] = rec_special_DIVU,
2492 [OP_SPECIAL_ADD] = rec_special_ADD,
2493 [OP_SPECIAL_ADDU] = rec_special_ADDU,
2494 [OP_SPECIAL_SUB] = rec_special_SUB,
2495 [OP_SPECIAL_SUBU] = rec_special_SUBU,
2496 [OP_SPECIAL_AND] = rec_special_AND,
2497 [OP_SPECIAL_OR] = rec_special_OR,
2498 [OP_SPECIAL_XOR] = rec_special_XOR,
2499 [OP_SPECIAL_NOR] = rec_special_NOR,
2500 [OP_SPECIAL_SLT] = rec_special_SLT,
2501 [OP_SPECIAL_SLTU] = rec_special_SLTU,
2504 static const lightrec_rec_func_t rec_regimm[64] = {
2505 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2506 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
2507 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
2508 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
2509 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
2512 static const lightrec_rec_func_t rec_cp0[64] = {
2513 SET_DEFAULT_ELM(rec_cp0, rec_CP),
2514 [OP_CP0_MFC0] = rec_cp0_MFC0,
2515 [OP_CP0_CFC0] = rec_cp0_CFC0,
2516 [OP_CP0_MTC0] = rec_cp0_MTC0,
2517 [OP_CP0_CTC0] = rec_cp0_CTC0,
2518 [OP_CP0_RFE] = rec_cp0_RFE,
2521 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2522 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2523 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
2524 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
2525 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
2526 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
2529 static void rec_SPECIAL(struct lightrec_cstate *state,
2530 const struct block *block, u16 offset)
2532 union code c = block->opcode_list[offset].c;
2533 lightrec_rec_func_t f = rec_special[c.r.op];
2535 if (!HAS_DEFAULT_ELM && unlikely(!f))
2536 unknown_opcode(state, block, offset);
2538 (*f)(state, block, offset);
2541 static void rec_REGIMM(struct lightrec_cstate *state,
2542 const struct block *block, u16 offset)
2544 union code c = block->opcode_list[offset].c;
2545 lightrec_rec_func_t f = rec_regimm[c.r.rt];
2547 if (!HAS_DEFAULT_ELM && unlikely(!f))
2548 unknown_opcode(state, block, offset);
2550 (*f)(state, block, offset);
2553 static void rec_CP0(struct lightrec_cstate *state,
2554 const struct block *block, u16 offset)
2556 union code c = block->opcode_list[offset].c;
2557 lightrec_rec_func_t f = rec_cp0[c.r.rs];
2559 if (!HAS_DEFAULT_ELM && unlikely(!f))
2560 rec_CP(state, block, offset);
2562 (*f)(state, block, offset);
2565 static void rec_CP2(struct lightrec_cstate *state,
2566 const struct block *block, u16 offset)
2568 union code c = block->opcode_list[offset].c;
2570 if (c.r.op == OP_CP2_BASIC) {
2571 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2573 if (HAS_DEFAULT_ELM || likely(f)) {
2574 (*f)(state, block, offset);
2579 rec_CP(state, block, offset);
2582 void lightrec_rec_opcode(struct lightrec_cstate *state,
2583 const struct block *block, u16 offset)
2585 struct regcache *reg_cache = state->reg_cache;
2586 struct lightrec_branch_target *target;
2587 const struct opcode *op = &block->opcode_list[offset];
2588 jit_state_t *_jit = block->_jit;
2589 lightrec_rec_func_t f;
2592 if (op_flag_sync(op->flags)) {
2594 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2597 lightrec_storeback_regs(reg_cache, _jit);
2598 lightrec_regcache_reset(reg_cache);
2600 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2601 target = &state->targets[state->nb_targets++];
2602 target->offset = offset;
2603 target->label = jit_indirect();
2606 if (likely(op->opcode)) {
2607 f = rec_standard[op->i.op];
2609 if (!HAS_DEFAULT_ELM && unlikely(!f))
2610 unknown_opcode(state, block, offset);
2612 (*f)(state, block, offset);
2615 if (OPT_EARLY_UNLOAD) {
2616 unload_offset = offset +
2617 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2619 lightrec_do_early_unload(state, block, unload_offset);