1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "disassembler.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
27 pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
28 block->opcode_list[offset].c.opcode,
29 block->pc + (offset << 2));
32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
33 const struct block *block, u16 offset,
34 s8 reg_new_pc, u32 imm, u8 ra_reg,
35 u32 link, bool update_cycles)
37 struct regcache *reg_cache = state->reg_cache;
38 u32 cycles = state->cycles;
39 jit_state_t *_jit = block->_jit;
40 const struct opcode *op = &block->opcode_list[offset],
41 *next = &block->opcode_list[offset + 1];
43 jit_note(__FILE__, __LINE__);
46 /* Update the $ra register */
47 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
48 jit_movi(link_reg, link);
49 lightrec_free_reg(reg_cache, link_reg);
53 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
54 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
56 jit_movi(reg_new_pc, imm);
59 if (has_delay_slot(op->c) &&
60 !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) {
61 cycles += lightrec_cycles_of_opcode(next->c);
63 /* Recompile the delay slot */
65 lightrec_rec_opcode(state, block, offset + 1);
68 /* Store back remaining registers */
69 lightrec_storeback_regs(reg_cache, _jit);
71 jit_movr(JIT_V0, reg_new_pc);
73 if (cycles && update_cycles) {
74 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
75 pr_debug("EOB: %u cycles\n", cycles);
78 if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1)
79 state->branches[state->nb_branches++] = jit_jmpi();
82 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
83 u16 offset, bool after_op)
85 struct regcache *reg_cache = state->reg_cache;
86 jit_state_t *_jit = block->_jit;
87 union code c = block->opcode_list[offset].c;
88 u32 cycles = state->cycles;
91 cycles -= lightrec_cycles_of_opcode(c);
93 lightrec_storeback_regs(reg_cache, _jit);
95 jit_movi(JIT_V0, block->pc + (offset << 2));
96 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
98 state->branches[state->nb_branches++] = jit_jmpi();
101 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
103 struct regcache *reg_cache = state->reg_cache;
104 jit_state_t *_jit = block->_jit;
105 const struct opcode *op = &block->opcode_list[offset],
106 *next = &block->opcode_list[offset + 1];
107 u8 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
109 /* If the source register is already mapped to JIT_R0 or JIT_R1, and the
110 * delay slot is a I/O operation, unload the register, since JIT_R0 and
111 * JIT_R1 are explicitely used by the I/O opcode generators. */
112 if ((rs == JIT_R0 || rs == JIT_R1) &&
113 !(op->flags & LIGHTREC_NO_DS) &&
114 opcode_is_io(next->c) &&
115 !(next->flags & (LIGHTREC_NO_INVALIDATE | LIGHTREC_DIRECT_IO))) {
116 lightrec_unload_reg(reg_cache, _jit, rs);
117 lightrec_free_reg(reg_cache, rs);
119 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
122 lightrec_lock_reg(reg_cache, _jit, rs);
127 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
129 u8 rs = get_jr_jalr_reg(state, block, offset);
131 _jit_name(block->_jit, __func__);
132 lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
135 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
137 u8 rs = get_jr_jalr_reg(state, block, offset);
138 union code c = block->opcode_list[offset].c;
140 _jit_name(block->_jit, __func__);
141 lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
142 get_branch_pc(block, offset, 2), true);
145 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
147 union code c = block->opcode_list[offset].c;
149 _jit_name(block->_jit, __func__);
150 lightrec_emit_end_of_block(state, block, offset, -1,
151 (block->pc & 0xf0000000) | (c.j.imm << 2),
155 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
157 union code c = block->opcode_list[offset].c;
159 _jit_name(block->_jit, __func__);
160 lightrec_emit_end_of_block(state, block, offset, -1,
161 (block->pc & 0xf0000000) | (c.j.imm << 2),
162 31, get_branch_pc(block, offset, 2), true);
165 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
166 jit_code_t code, u32 link, bool unconditional, bool bz)
168 struct regcache *reg_cache = state->reg_cache;
169 struct native_register *regs_backup;
170 jit_state_t *_jit = block->_jit;
171 struct lightrec_branch *branch;
172 const struct opcode *op = &block->opcode_list[offset],
173 *next = &block->opcode_list[offset + 1];
176 u32 target_offset, cycles = state->cycles;
177 bool is_forward = (s16)op->i.imm >= -1;
180 jit_note(__FILE__, __LINE__);
182 if (!(op->flags & LIGHTREC_NO_DS))
183 cycles += lightrec_cycles_of_opcode(next->c);
188 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
190 if (!unconditional) {
191 u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT),
192 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
193 _jit, op->i.rt, REG_EXT);
195 /* Generate the branch opcode */
196 addr = jit_new_node_pww(code, NULL, rs, rt);
198 lightrec_free_regs(reg_cache);
199 regs_backup = lightrec_regcache_enter_branch(reg_cache);
202 if (op->flags & LIGHTREC_LOCAL_BRANCH) {
203 if (next && !(op->flags & LIGHTREC_NO_DS)) {
204 /* Recompile the delay slot */
206 lightrec_rec_opcode(state, block, offset + 1);
210 /* Update the $ra register */
211 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
212 jit_movi(link_reg, link);
213 lightrec_free_reg(reg_cache, link_reg);
216 /* Store back remaining registers */
217 lightrec_storeback_regs(reg_cache, _jit);
219 target_offset = offset + 1 + (s16)op->i.imm
220 - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS));
221 pr_debug("Adding local branch to offset 0x%x\n",
223 branch = &state->local_branches[
224 state->nb_local_branches++];
226 branch->target = target_offset;
228 branch->branch = jit_jmpi();
230 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
233 if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) {
234 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
235 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
239 if (!unconditional) {
241 lightrec_regcache_leave_branch(reg_cache, regs_backup);
244 /* Update the $ra register */
245 link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
247 jit_movi(link_reg, (s32)link);
248 lightrec_free_reg(reg_cache, link_reg);
251 if (!(op->flags & LIGHTREC_NO_DS) && next->opcode)
252 lightrec_rec_opcode(state, block, offset + 1);
256 static void rec_BNE(struct lightrec_cstate *state,
257 const struct block *block, u16 offset)
259 union code c = block->opcode_list[offset].c;
261 _jit_name(block->_jit, __func__);
264 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
266 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
269 static void rec_BEQ(struct lightrec_cstate *state,
270 const struct block *block, u16 offset)
272 union code c = block->opcode_list[offset].c;
274 _jit_name(block->_jit, __func__);
277 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
279 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
282 static void rec_BLEZ(struct lightrec_cstate *state,
283 const struct block *block, u16 offset)
285 union code c = block->opcode_list[offset].c;
287 _jit_name(block->_jit, __func__);
288 rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
291 static void rec_BGTZ(struct lightrec_cstate *state,
292 const struct block *block, u16 offset)
294 _jit_name(block->_jit, __func__);
295 rec_b(state, block, offset, jit_code_blei, 0, false, true);
298 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
299 const struct block *block, u16 offset)
301 _jit_name(block->_jit, __func__);
302 rec_b(state, block, offset, jit_code_bgei, 0, false, true);
305 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
306 const struct block *block, u16 offset)
308 _jit_name(block->_jit, __func__);
309 rec_b(state, block, offset, jit_code_bgei,
310 get_branch_pc(block, offset, 2), false, true);
313 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
314 const struct block *block, u16 offset)
316 union code c = block->opcode_list[offset].c;
318 _jit_name(block->_jit, __func__);
319 rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
322 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
323 const struct block *block, u16 offset)
325 const struct opcode *op = &block->opcode_list[offset];
326 _jit_name(block->_jit, __func__);
327 rec_b(state, block, offset, jit_code_blti,
328 get_branch_pc(block, offset, 2),
332 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
333 u16 offset, jit_code_t code, bool slti)
335 struct regcache *reg_cache = state->reg_cache;
336 union code c = block->opcode_list[offset].c;
337 jit_state_t *_jit = block->_jit;
338 u8 rs, rt, out_flags = REG_EXT;
341 out_flags |= REG_ZEXT;
343 jit_note(__FILE__, __LINE__);
344 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
345 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
347 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
349 lightrec_free_reg(reg_cache, rs);
350 lightrec_free_reg(reg_cache, rt);
353 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
354 u16 offset, jit_code_t code, bool out_ext)
356 struct regcache *reg_cache = state->reg_cache;
357 union code c = block->opcode_list[offset].c;
358 jit_state_t *_jit = block->_jit;
361 jit_note(__FILE__, __LINE__);
362 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
363 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
364 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
365 out_ext ? REG_EXT | REG_ZEXT : 0);
367 jit_new_node_www(code, rd, rs, rt);
369 lightrec_free_reg(reg_cache, rs);
370 lightrec_free_reg(reg_cache, rt);
371 lightrec_free_reg(reg_cache, rd);
374 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
375 u16 offset, jit_code_t code)
377 struct regcache *reg_cache = state->reg_cache;
378 union code c = block->opcode_list[offset].c;
379 jit_state_t *_jit = block->_jit;
380 u8 rd, rt, rs, temp, flags = 0;
382 jit_note(__FILE__, __LINE__);
383 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
385 if (code == jit_code_rshr)
387 else if (code == jit_code_rshr_u)
390 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
391 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
393 if (rs != rd && rt != rd) {
394 jit_andi(rd, rs, 0x1f);
395 jit_new_node_www(code, rd, rt, rd);
397 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
398 jit_andi(temp, rs, 0x1f);
399 jit_new_node_www(code, rd, rt, temp);
400 lightrec_free_reg(reg_cache, temp);
403 lightrec_free_reg(reg_cache, rs);
404 lightrec_free_reg(reg_cache, rt);
405 lightrec_free_reg(reg_cache, rd);
408 static void rec_ADDIU(struct lightrec_cstate *state,
409 const struct block *block, u16 offset)
411 _jit_name(block->_jit, __func__);
412 rec_alu_imm(state, block, offset, jit_code_addi, false);
415 static void rec_ADDI(struct lightrec_cstate *state,
416 const struct block *block, u16 offset)
418 /* TODO: Handle the exception? */
419 _jit_name(block->_jit, __func__);
420 rec_alu_imm(state, block, offset, jit_code_addi, false);
423 static void rec_SLTIU(struct lightrec_cstate *state,
424 const struct block *block, u16 offset)
426 _jit_name(block->_jit, __func__);
427 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
430 static void rec_SLTI(struct lightrec_cstate *state,
431 const struct block *block, u16 offset)
433 _jit_name(block->_jit, __func__);
434 rec_alu_imm(state, block, offset, jit_code_lti, true);
437 static void rec_ANDI(struct lightrec_cstate *state,
438 const struct block *block, u16 offset)
440 struct regcache *reg_cache = state->reg_cache;
441 union code c = block->opcode_list[offset].c;
442 jit_state_t *_jit = block->_jit;
445 _jit_name(block->_jit, __func__);
446 jit_note(__FILE__, __LINE__);
447 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
448 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
451 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
452 * casts to uint8_t / uint16_t. */
455 else if (c.i.imm == 0xffff)
458 jit_andi(rt, rs, (u32)(u16) c.i.imm);
460 lightrec_free_reg(reg_cache, rs);
461 lightrec_free_reg(reg_cache, rt);
464 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
465 u16 offset, jit_code_t code)
467 struct regcache *reg_cache = state->reg_cache;
468 union code c = block->opcode_list[offset].c;
469 jit_state_t *_jit = block->_jit;
472 jit_note(__FILE__, __LINE__);
473 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
474 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
476 flags = lightrec_get_reg_in_flags(reg_cache, rs);
477 lightrec_set_reg_out_flags(reg_cache, rt, flags);
479 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
481 lightrec_free_reg(reg_cache, rs);
482 lightrec_free_reg(reg_cache, rt);
486 static void rec_ORI(struct lightrec_cstate *state,
487 const struct block *block, u16 offset)
489 _jit_name(block->_jit, __func__);
490 rec_alu_or_xor(state, block, offset, jit_code_ori);
493 static void rec_XORI(struct lightrec_cstate *state,
494 const struct block *block, u16 offset)
496 _jit_name(block->_jit, __func__);
497 rec_alu_or_xor(state, block, offset, jit_code_xori);
500 static void rec_LUI(struct lightrec_cstate *state,
501 const struct block *block, u16 offset)
503 struct regcache *reg_cache = state->reg_cache;
504 union code c = block->opcode_list[offset].c;
505 jit_state_t *_jit = block->_jit;
506 u8 rt, flags = REG_EXT;
509 jit_note(__FILE__, __LINE__);
511 if (!(c.i.imm & BIT(15)))
514 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
516 jit_movi(rt, (s32)(c.i.imm << 16));
518 lightrec_free_reg(reg_cache, rt);
521 static void rec_special_ADDU(struct lightrec_cstate *state,
522 const struct block *block, u16 offset)
524 _jit_name(block->_jit, __func__);
525 rec_alu_special(state, block, offset, jit_code_addr, false);
528 static void rec_special_ADD(struct lightrec_cstate *state,
529 const struct block *block, u16 offset)
531 /* TODO: Handle the exception? */
532 _jit_name(block->_jit, __func__);
533 rec_alu_special(state, block, offset, jit_code_addr, false);
536 static void rec_special_SUBU(struct lightrec_cstate *state,
537 const struct block *block, u16 offset)
539 _jit_name(block->_jit, __func__);
540 rec_alu_special(state, block, offset, jit_code_subr, false);
543 static void rec_special_SUB(struct lightrec_cstate *state,
544 const struct block *block, u16 offset)
546 /* TODO: Handle the exception? */
547 _jit_name(block->_jit, __func__);
548 rec_alu_special(state, block, offset, jit_code_subr, false);
551 static void rec_special_AND(struct lightrec_cstate *state,
552 const struct block *block, u16 offset)
554 struct regcache *reg_cache = state->reg_cache;
555 union code c = block->opcode_list[offset].c;
556 jit_state_t *_jit = block->_jit;
557 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
559 _jit_name(block->_jit, __func__);
560 jit_note(__FILE__, __LINE__);
561 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
562 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
563 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
565 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
566 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
568 /* Z(rd) = Z(rs) | Z(rt) */
569 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
571 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
572 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
573 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
574 (REG_EXT & flags_rs & flags_rt))
577 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
579 jit_andr(rd, rs, rt);
581 lightrec_free_reg(reg_cache, rs);
582 lightrec_free_reg(reg_cache, rt);
583 lightrec_free_reg(reg_cache, rd);
586 static void rec_special_or_nor(struct lightrec_cstate *state,
587 const struct block *block, u16 offset, bool nor)
589 struct regcache *reg_cache = state->reg_cache;
590 union code c = block->opcode_list[offset].c;
591 jit_state_t *_jit = block->_jit;
592 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
594 jit_note(__FILE__, __LINE__);
595 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
596 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
597 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
599 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
600 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
602 /* or: Z(rd) = Z(rs) & Z(rt)
605 flags_rd = REG_ZEXT & flags_rs & flags_rt;
607 /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
608 if ((REG_EXT & flags_rs & flags_rt) ||
609 (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
610 (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
613 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
620 lightrec_free_reg(reg_cache, rs);
621 lightrec_free_reg(reg_cache, rt);
622 lightrec_free_reg(reg_cache, rd);
625 static void rec_special_OR(struct lightrec_cstate *state,
626 const struct block *block, u16 offset)
628 _jit_name(block->_jit, __func__);
629 rec_special_or_nor(state, block, offset, false);
632 static void rec_special_NOR(struct lightrec_cstate *state,
633 const struct block *block, u16 offset)
635 _jit_name(block->_jit, __func__);
636 rec_special_or_nor(state, block, offset, true);
639 static void rec_special_XOR(struct lightrec_cstate *state,
640 const struct block *block, u16 offset)
642 struct regcache *reg_cache = state->reg_cache;
643 union code c = block->opcode_list[offset].c;
644 jit_state_t *_jit = block->_jit;
645 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
647 _jit_name(block->_jit, __func__);
649 jit_note(__FILE__, __LINE__);
650 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
651 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
652 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
654 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
655 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
657 /* Z(rd) = Z(rs) & Z(rt) */
658 flags_rd = REG_ZEXT & flags_rs & flags_rt;
660 /* E(rd) = E(rs) & E(rt) */
661 flags_rd |= REG_EXT & flags_rs & flags_rt;
663 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
665 jit_xorr(rd, rs, rt);
667 lightrec_free_reg(reg_cache, rs);
668 lightrec_free_reg(reg_cache, rt);
669 lightrec_free_reg(reg_cache, rd);
672 static void rec_special_SLTU(struct lightrec_cstate *state,
673 const struct block *block, u16 offset)
675 _jit_name(block->_jit, __func__);
676 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
679 static void rec_special_SLT(struct lightrec_cstate *state,
680 const struct block *block, u16 offset)
682 _jit_name(block->_jit, __func__);
683 rec_alu_special(state, block, offset, jit_code_ltr, true);
686 static void rec_special_SLLV(struct lightrec_cstate *state,
687 const struct block *block, u16 offset)
689 _jit_name(block->_jit, __func__);
690 rec_alu_shiftv(state, block, offset, jit_code_lshr);
693 static void rec_special_SRLV(struct lightrec_cstate *state,
694 const struct block *block, u16 offset)
696 _jit_name(block->_jit, __func__);
697 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
700 static void rec_special_SRAV(struct lightrec_cstate *state,
701 const struct block *block, u16 offset)
703 _jit_name(block->_jit, __func__);
704 rec_alu_shiftv(state, block, offset, jit_code_rshr);
707 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
708 u16 offset, jit_code_t code)
710 struct regcache *reg_cache = state->reg_cache;
711 union code c = block->opcode_list[offset].c;
712 jit_state_t *_jit = block->_jit;
713 u8 rd, rt, flags = 0;
715 jit_note(__FILE__, __LINE__);
717 if (code == jit_code_rshi)
719 else if (code == jit_code_rshi_u)
722 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
724 /* Input reg is zero-extended, if we SRL at least by one bit, we know
725 * the output reg will be both zero-extended and sign-extended. */
726 if (code == jit_code_rshi_u && c.r.imm)
728 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
730 jit_new_node_www(code, rd, rt, c.r.imm);
732 lightrec_free_reg(reg_cache, rt);
733 lightrec_free_reg(reg_cache, rd);
736 static void rec_special_SLL(struct lightrec_cstate *state,
737 const struct block *block, u16 offset)
739 _jit_name(block->_jit, __func__);
740 rec_alu_shift(state, block, offset, jit_code_lshi);
743 static void rec_special_SRL(struct lightrec_cstate *state,
744 const struct block *block, u16 offset)
746 _jit_name(block->_jit, __func__);
747 rec_alu_shift(state, block, offset, jit_code_rshi_u);
750 static void rec_special_SRA(struct lightrec_cstate *state,
751 const struct block *block, u16 offset)
753 _jit_name(block->_jit, __func__);
754 rec_alu_shift(state, block, offset, jit_code_rshi);
757 static void rec_alu_mult(struct lightrec_cstate *state,
758 const struct block *block, u16 offset, bool is_signed)
760 struct regcache *reg_cache = state->reg_cache;
761 union code c = block->opcode_list[offset].c;
762 u16 flags = block->opcode_list[offset].flags;
763 u8 reg_lo = get_mult_div_lo(c);
764 u8 reg_hi = get_mult_div_hi(c);
765 jit_state_t *_jit = block->_jit;
766 u8 lo, hi, rs, rt, rflags = 0;
768 jit_note(__FILE__, __LINE__);
775 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
776 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
778 if (!(flags & LIGHTREC_NO_LO))
779 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
780 else if (__WORDSIZE == 32)
781 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
783 if (!(flags & LIGHTREC_NO_HI))
784 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
786 if (__WORDSIZE == 32) {
787 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
788 * operation if the MULT was detected a 32-bit only. */
789 if (!(flags & LIGHTREC_NO_HI)) {
791 jit_qmulr(lo, hi, rs, rt);
793 jit_qmulr_u(lo, hi, rs, rt);
795 jit_mulr(lo, rs, rt);
798 /* On 64-bit systems, do a 64*64->64 bit operation. */
799 if (flags & LIGHTREC_NO_LO) {
800 jit_mulr(hi, rs, rt);
801 jit_rshi(hi, hi, 32);
803 jit_mulr(lo, rs, rt);
805 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
806 if (!(flags & LIGHTREC_NO_HI))
807 jit_rshi(hi, lo, 32);
811 lightrec_free_reg(reg_cache, rs);
812 lightrec_free_reg(reg_cache, rt);
813 if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32)
814 lightrec_free_reg(reg_cache, lo);
815 if (!(flags & LIGHTREC_NO_HI))
816 lightrec_free_reg(reg_cache, hi);
819 static void rec_alu_div(struct lightrec_cstate *state,
820 const struct block *block, u16 offset, bool is_signed)
822 struct regcache *reg_cache = state->reg_cache;
823 union code c = block->opcode_list[offset].c;
824 u16 flags = block->opcode_list[offset].flags;
825 bool no_check = flags & LIGHTREC_NO_DIV_CHECK;
826 u8 reg_lo = get_mult_div_lo(c);
827 u8 reg_hi = get_mult_div_hi(c);
828 jit_state_t *_jit = block->_jit;
829 jit_node_t *branch, *to_end;
830 u8 lo, hi, rs, rt, rflags = 0;
832 jit_note(__FILE__, __LINE__);
839 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
840 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
842 if (!(flags & LIGHTREC_NO_LO))
843 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
845 if (!(flags & LIGHTREC_NO_HI))
846 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
848 /* Jump to special handler if dividing by zero */
850 branch = jit_beqi(rt, 0);
852 if (flags & LIGHTREC_NO_LO) {
854 jit_remr(hi, rs, rt);
856 jit_remr_u(hi, rs, rt);
857 } else if (flags & LIGHTREC_NO_HI) {
859 jit_divr(lo, rs, rt);
861 jit_divr_u(lo, rs, rt);
864 jit_qdivr(lo, hi, rs, rt);
866 jit_qdivr_u(lo, hi, rs, rt);
870 lightrec_regcache_mark_live(reg_cache, _jit);
872 /* Jump above the div-by-zero handler */
877 if (!(flags & LIGHTREC_NO_LO)) {
883 jit_movi(lo, 0xffffffff);
887 if (!(flags & LIGHTREC_NO_HI))
893 lightrec_free_reg(reg_cache, rs);
894 lightrec_free_reg(reg_cache, rt);
896 if (!(flags & LIGHTREC_NO_LO))
897 lightrec_free_reg(reg_cache, lo);
899 if (!(flags & LIGHTREC_NO_HI))
900 lightrec_free_reg(reg_cache, hi);
903 static void rec_special_MULT(struct lightrec_cstate *state,
904 const struct block *block, u16 offset)
906 _jit_name(block->_jit, __func__);
907 rec_alu_mult(state, block, offset, true);
910 static void rec_special_MULTU(struct lightrec_cstate *state,
911 const struct block *block, u16 offset)
913 _jit_name(block->_jit, __func__);
914 rec_alu_mult(state, block, offset, false);
917 static void rec_special_DIV(struct lightrec_cstate *state,
918 const struct block *block, u16 offset)
920 _jit_name(block->_jit, __func__);
921 rec_alu_div(state, block, offset, true);
924 static void rec_special_DIVU(struct lightrec_cstate *state,
925 const struct block *block, u16 offset)
927 _jit_name(block->_jit, __func__);
928 rec_alu_div(state, block, offset, false);
931 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
932 const struct block *block, u8 dst, u8 src)
934 struct regcache *reg_cache = state->reg_cache;
935 jit_state_t *_jit = block->_jit;
937 jit_note(__FILE__, __LINE__);
938 src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
939 dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
941 jit_extr_i(dst, src);
943 lightrec_free_reg(reg_cache, src);
944 lightrec_free_reg(reg_cache, dst);
947 static void rec_special_MFHI(struct lightrec_cstate *state,
948 const struct block *block, u16 offset)
950 union code c = block->opcode_list[offset].c;
952 _jit_name(block->_jit, __func__);
953 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
956 static void rec_special_MTHI(struct lightrec_cstate *state,
957 const struct block *block, u16 offset)
959 union code c = block->opcode_list[offset].c;
961 _jit_name(block->_jit, __func__);
962 rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
965 static void rec_special_MFLO(struct lightrec_cstate *state,
966 const struct block *block, u16 offset)
968 union code c = block->opcode_list[offset].c;
970 _jit_name(block->_jit, __func__);
971 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
974 static void rec_special_MTLO(struct lightrec_cstate *state,
975 const struct block *block, u16 offset)
977 union code c = block->opcode_list[offset].c;
979 _jit_name(block->_jit, __func__);
980 rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
983 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
984 u32 arg, bool with_arg, enum c_wrappers wrapper)
986 struct regcache *reg_cache = state->reg_cache;
987 jit_state_t *_jit = block->_jit;
991 tmp3 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
992 tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R0);
993 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
995 jit_ldxi(tmp, LIGHTREC_REG_STATE,
996 offsetof(struct lightrec_state, c_wrapper));
997 jit_ldxi(tmp2, LIGHTREC_REG_STATE,
998 offsetof(struct lightrec_state, c_wrappers[wrapper]));
1000 jit_movi(tmp3, arg);
1004 lightrec_free_reg(reg_cache, tmp);
1005 lightrec_free_reg(reg_cache, tmp2);
1007 lightrec_free_reg(reg_cache, tmp3);
1008 lightrec_regcache_mark_live(reg_cache, _jit);
1011 static void rec_io(struct lightrec_cstate *state,
1012 const struct block *block, u16 offset,
1013 bool load_rt, bool read_rt)
1015 struct regcache *reg_cache = state->reg_cache;
1016 jit_state_t *_jit = block->_jit;
1017 union code c = block->opcode_list[offset].c;
1018 u16 flags = block->opcode_list[offset].flags;
1019 bool is_tagged = flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
1022 jit_note(__FILE__, __LINE__);
1024 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1026 if (read_rt && likely(c.i.rt))
1027 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1029 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1032 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1034 lut_entry = lightrec_get_lut_entry(block);
1035 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1036 true, C_WRAPPER_RW_GENERIC);
1040 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1041 const struct block *block,
1042 u16 offset, jit_code_t code)
1044 struct lightrec_state *state = cstate->state;
1045 struct regcache *reg_cache = cstate->reg_cache;
1046 union code c = block->opcode_list[offset].c;
1047 jit_state_t *_jit = block->_jit;
1048 jit_node_t *to_not_ram, *to_end;
1049 u8 tmp, tmp2, rs, rt;
1052 jit_note(__FILE__, __LINE__);
1053 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1054 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1056 if (state->offset_ram || state->offset_scratch)
1057 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1059 /* Convert to KUNSEG and avoid RAM mirrors */
1060 if (state->mirrors_mapped) {
1062 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1063 } else if (c.i.imm) {
1065 jit_addi(tmp, rs, (s16)c.i.imm);
1066 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1069 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1072 lightrec_free_reg(reg_cache, rs);
1074 if (state->offset_ram != state->offset_scratch) {
1075 to_not_ram = jit_bmsi(tmp, BIT(28));
1077 lightrec_regcache_mark_live(reg_cache, _jit);
1079 jit_movi(tmp2, state->offset_ram);
1081 to_end = jit_jmpi();
1082 jit_patch(to_not_ram);
1084 jit_movi(tmp2, state->offset_scratch);
1086 } else if (state->offset_ram) {
1087 jit_movi(tmp2, state->offset_ram);
1090 if (state->offset_ram || state->offset_scratch) {
1091 jit_addr(tmp, tmp, tmp2);
1092 lightrec_free_reg(reg_cache, tmp2);
1095 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1096 jit_new_node_www(code, imm, tmp, rt);
1098 lightrec_free_reg(reg_cache, rt);
1099 lightrec_free_reg(reg_cache, tmp);
1102 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1103 u16 offset, jit_code_t code)
1105 struct lightrec_state *state = cstate->state;
1106 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1107 struct regcache *reg_cache = cstate->reg_cache;
1108 union code c = block->opcode_list[offset].c;
1109 jit_state_t *_jit = block->_jit;
1110 jit_node_t *to_not_ram, *to_end;
1111 u8 tmp, tmp2, tmp3, rs, rt;
1113 jit_note(__FILE__, __LINE__);
1115 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1116 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1117 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1119 /* Convert to KUNSEG and avoid RAM mirrors */
1121 jit_addi(tmp2, rs, (s16)c.i.imm);
1122 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1124 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1127 lightrec_free_reg(reg_cache, rs);
1128 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1130 to_not_ram = jit_bgti(tmp2, ram_size);
1132 lightrec_regcache_mark_live(reg_cache, _jit);
1134 /* Compute the offset to the code LUT */
1135 jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1136 if (__WORDSIZE == 64)
1137 jit_lshi(tmp, tmp, 1);
1138 jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1140 /* Write NULL to the code LUT to invalidate any block that's there */
1141 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1143 if (state->offset_ram != state->offset_scratch) {
1144 jit_movi(tmp, state->offset_ram);
1146 to_end = jit_jmpi();
1149 jit_patch(to_not_ram);
1151 if (state->offset_ram || state->offset_scratch)
1152 jit_movi(tmp, state->offset_scratch);
1154 if (state->offset_ram != state->offset_scratch)
1157 if (state->offset_ram || state->offset_scratch)
1158 jit_addr(tmp2, tmp2, tmp);
1160 lightrec_free_reg(reg_cache, tmp);
1161 lightrec_free_reg(reg_cache, tmp3);
1163 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1164 jit_new_node_www(code, 0, tmp2, rt);
1166 lightrec_free_reg(reg_cache, rt);
1167 lightrec_free_reg(reg_cache, tmp2);
1170 static void rec_store(struct lightrec_cstate *state,
1171 const struct block *block, u16 offset, jit_code_t code)
1173 u16 flags = block->opcode_list[offset].flags;
1175 if (flags & LIGHTREC_NO_INVALIDATE) {
1176 rec_store_direct_no_invalidate(state, block, offset, code);
1177 } else if (flags & LIGHTREC_DIRECT_IO) {
1178 if (state->state->invalidate_from_dma_only)
1179 rec_store_direct_no_invalidate(state, block, offset, code);
1181 rec_store_direct(state, block, offset, code);
1183 rec_io(state, block, offset, true, false);
1187 static void rec_SB(struct lightrec_cstate *state,
1188 const struct block *block, u16 offset)
1190 _jit_name(block->_jit, __func__);
1191 rec_store(state, block, offset, jit_code_stxi_c);
1194 static void rec_SH(struct lightrec_cstate *state,
1195 const struct block *block, u16 offset)
1197 _jit_name(block->_jit, __func__);
1198 rec_store(state, block, offset, jit_code_stxi_s);
1201 static void rec_SW(struct lightrec_cstate *state,
1202 const struct block *block, u16 offset)
1205 _jit_name(block->_jit, __func__);
1206 rec_store(state, block, offset, jit_code_stxi_i);
1209 static void rec_SWL(struct lightrec_cstate *state,
1210 const struct block *block, u16 offset)
1212 _jit_name(block->_jit, __func__);
1213 rec_io(state, block, offset, true, false);
1216 static void rec_SWR(struct lightrec_cstate *state,
1217 const struct block *block, u16 offset)
1219 _jit_name(block->_jit, __func__);
1220 rec_io(state, block, offset, true, false);
1223 static void rec_SWC2(struct lightrec_cstate *state,
1224 const struct block *block, u16 offset)
1226 _jit_name(block->_jit, __func__);
1227 rec_io(state, block, offset, false, false);
1230 static void rec_load_direct(struct lightrec_cstate *cstate, const struct block *block,
1231 u16 offset, jit_code_t code, bool is_unsigned)
1233 struct lightrec_state *state = cstate->state;
1234 struct regcache *reg_cache = cstate->reg_cache;
1235 union code c = block->opcode_list[offset].c;
1236 jit_state_t *_jit = block->_jit;
1237 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1238 u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1247 jit_note(__FILE__, __LINE__);
1248 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1249 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1251 if ((state->offset_ram == state->offset_bios &&
1252 state->offset_ram == state->offset_scratch &&
1253 state->mirrors_mapped) || !c.i.imm) {
1257 jit_addi(rt, rs, (s16)c.i.imm);
1261 if (c.i.rs != c.i.rt)
1262 lightrec_free_reg(reg_cache, rs);
1265 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1267 if (state->offset_ram == state->offset_bios &&
1268 state->offset_ram == state->offset_scratch) {
1269 if (!state->mirrors_mapped) {
1270 jit_andi(tmp, addr_reg, BIT(28));
1271 jit_rshi_u(tmp, tmp, 28 - 22);
1272 jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1273 jit_andr(rt, addr_reg, tmp);
1275 jit_andi(rt, addr_reg, 0x1fffffff);
1278 if (state->offset_ram)
1279 jit_movi(tmp, state->offset_ram);
1281 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1283 lightrec_regcache_mark_live(reg_cache, _jit);
1285 /* Convert to KUNSEG and avoid RAM mirrors */
1286 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1288 if (state->offset_ram)
1289 jit_movi(tmp, state->offset_ram);
1291 to_end = jit_jmpi();
1293 jit_patch(to_not_ram);
1295 if (state->offset_bios != state->offset_scratch)
1296 to_not_bios = jit_bmci(addr_reg, BIT(22));
1298 /* Convert to KUNSEG */
1299 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1301 jit_movi(tmp, state->offset_bios);
1303 if (state->offset_bios != state->offset_scratch) {
1304 to_end2 = jit_jmpi();
1306 jit_patch(to_not_bios);
1308 /* Convert to KUNSEG */
1309 jit_andi(rt, addr_reg, 0x1f800fff);
1311 if (state->offset_scratch)
1312 jit_movi(tmp, state->offset_scratch);
1320 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1321 jit_addr(rt, rt, tmp);
1323 jit_new_node_www(code, rt, rt, imm);
1325 lightrec_free_reg(reg_cache, addr_reg);
1326 lightrec_free_reg(reg_cache, rt);
1327 lightrec_free_reg(reg_cache, tmp);
1330 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1331 u16 offset, jit_code_t code, bool is_unsigned)
1333 u16 flags = block->opcode_list[offset].flags;
1335 if (flags & LIGHTREC_DIRECT_IO)
1336 rec_load_direct(state, block, offset, code, is_unsigned);
1338 rec_io(state, block, offset, false, true);
1341 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1343 _jit_name(block->_jit, __func__);
1344 rec_load(state, block, offset, jit_code_ldxi_c, false);
1347 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1349 _jit_name(block->_jit, __func__);
1350 rec_load(state, block, offset, jit_code_ldxi_uc, true);
1353 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1355 _jit_name(block->_jit, __func__);
1356 rec_load(state, block, offset, jit_code_ldxi_s, false);
1359 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1361 _jit_name(block->_jit, __func__);
1362 rec_load(state, block, offset, jit_code_ldxi_us, true);
1365 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1367 _jit_name(block->_jit, __func__);
1368 rec_io(state, block, offset, true, true);
1371 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1373 _jit_name(block->_jit, __func__);
1374 rec_io(state, block, offset, true, true);
1377 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1379 _jit_name(block->_jit, __func__);
1380 rec_load(state, block, offset, jit_code_ldxi_i, false);
1383 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1385 _jit_name(block->_jit, __func__);
1386 rec_io(state, block, offset, false, false);
1389 static void rec_break_syscall(struct lightrec_cstate *state,
1390 const struct block *block, u16 offset, bool is_break)
1392 _jit_note(block->_jit, __FILE__, __LINE__);
1395 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1397 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1399 /* TODO: the return address should be "pc - 4" if we're a delay slot */
1400 lightrec_emit_end_of_block(state, block, offset, -1,
1401 get_ds_pc(block, offset, 0),
1405 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1406 const struct block *block, u16 offset)
1408 _jit_name(block->_jit, __func__);
1409 rec_break_syscall(state, block, offset, false);
1412 static void rec_special_BREAK(struct lightrec_cstate *state,
1413 const struct block *block, u16 offset)
1415 _jit_name(block->_jit, __func__);
1416 rec_break_syscall(state, block, offset, true);
1419 static void rec_mfc(struct lightrec_cstate *state,
1420 const struct block *block, u16 offset)
1422 struct regcache *reg_cache = state->reg_cache;
1423 union code c = block->opcode_list[offset].c;
1424 jit_state_t *_jit = block->_jit;
1426 jit_note(__FILE__, __LINE__);
1427 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1429 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MFC);
1432 static void rec_mtc(struct lightrec_cstate *state,
1433 const struct block *block, u16 offset)
1435 struct regcache *reg_cache = state->reg_cache;
1436 union code c = block->opcode_list[offset].c;
1437 jit_state_t *_jit = block->_jit;
1439 jit_note(__FILE__, __LINE__);
1440 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1441 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1443 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1445 if (c.i.op == OP_CP0 &&
1446 !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1447 (c.r.rd == 12 || c.r.rd == 13))
1448 lightrec_emit_end_of_block(state, block, offset, -1,
1449 get_ds_pc(block, offset, 1),
1454 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1456 struct regcache *reg_cache = state->reg_cache;
1457 union code c = block->opcode_list[offset].c;
1458 jit_state_t *_jit = block->_jit;
1461 jit_note(__FILE__, __LINE__);
1463 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1465 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1466 offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1468 lightrec_free_reg(reg_cache, rt);
1471 static bool block_in_bios(const struct lightrec_cstate *state,
1472 const struct block *block)
1474 const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1475 u32 pc = kunseg(block->pc);
1477 return pc >= bios->pc && pc < bios->pc + bios->length;
1481 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1483 struct regcache *reg_cache = state->reg_cache;
1484 const union code c = block->opcode_list[offset].c;
1485 jit_state_t *_jit = block->_jit;
1486 u8 rt, tmp, tmp2, status;
1488 jit_note(__FILE__, __LINE__);
1496 /* Those registers are read-only */
1502 if (block_in_bios(state, block) && c.r.rd == 12) {
1503 /* If we are running code from the BIOS, handle writes to the
1504 * Status register in C. BIOS code may toggle bit 16 which will
1505 * map/unmap the RAM, while game code cannot do that. */
1506 rec_mtc(state, block, offset);
1510 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1513 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1514 LIGHTREC_REG_STATE, rt);
1517 if (c.r.rd == 12 || c.r.rd == 13) {
1518 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1519 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1520 offsetof(struct lightrec_state, regs.cp0[13]));
1525 } else if (c.r.rd == 13) {
1526 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1528 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1529 jit_andi(tmp2, rt, 0x0300);
1530 jit_ori(tmp, tmp, 0x0300);
1531 jit_xori(tmp, tmp, 0x0300);
1532 jit_orr(tmp, tmp, tmp2);
1533 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1534 offsetof(struct lightrec_state, regs.cp0[12]));
1535 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1536 LIGHTREC_REG_STATE, tmp);
1540 if (c.r.rd == 12 || c.r.rd == 13) {
1541 /* Exit dynarec in case there's a software interrupt.
1542 * exit_flags = !!(status & tmp & 0x0300) & status; */
1543 jit_andr(tmp, tmp, status);
1544 jit_andi(tmp, tmp, 0x0300);
1545 jit_nei(tmp, tmp, 0);
1546 jit_andr(tmp, tmp, status);
1547 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1548 LIGHTREC_REG_STATE, tmp);
1550 lightrec_free_reg(reg_cache, tmp);
1554 lightrec_free_reg(reg_cache, tmp2);
1556 lightrec_free_reg(reg_cache, rt);
1558 if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1559 (c.r.rd == 12 || c.r.rd == 13))
1560 lightrec_emit_eob(state, block, offset + 1, true);
1563 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1564 const struct block *block, u16 offset)
1566 _jit_name(block->_jit, __func__);
1567 rec_mfc0(state, block, offset);
1570 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1571 const struct block *block, u16 offset)
1573 _jit_name(block->_jit, __func__);
1574 rec_mfc0(state, block, offset);
1577 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1578 const struct block *block, u16 offset)
1580 _jit_name(block->_jit, __func__);
1581 rec_mtc0(state, block, offset);
1584 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1585 const struct block *block, u16 offset)
1587 _jit_name(block->_jit, __func__);
1588 rec_mtc0(state, block, offset);
1591 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1592 const struct block *block, u16 offset)
1594 _jit_name(block->_jit, __func__);
1595 rec_mfc(state, block, offset);
1598 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
1599 const struct block *block, u16 offset)
1601 _jit_name(block->_jit, __func__);
1602 rec_mfc(state, block, offset);
1605 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
1606 const struct block *block, u16 offset)
1608 _jit_name(block->_jit, __func__);
1609 rec_mtc(state, block, offset);
1612 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
1613 const struct block *block, u16 offset)
1615 _jit_name(block->_jit, __func__);
1616 rec_mtc(state, block, offset);
1619 static void rec_cp0_RFE(struct lightrec_cstate *state,
1620 const struct block *block, u16 offset)
1622 struct regcache *reg_cache = state->reg_cache;
1623 jit_state_t *_jit = block->_jit;
1627 jit_note(__FILE__, __LINE__);
1629 status = lightrec_alloc_reg_temp(reg_cache, _jit);
1630 jit_ldxi_i(status, LIGHTREC_REG_STATE,
1631 offsetof(struct lightrec_state, regs.cp0[12]));
1633 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1635 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
1636 jit_rshi(tmp, status, 2);
1637 jit_andi(tmp, tmp, 0xf);
1638 jit_andi(status, status, ~0xful);
1639 jit_orr(status, status, tmp);
1641 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1642 offsetof(struct lightrec_state, regs.cp0[13]));
1643 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
1644 LIGHTREC_REG_STATE, status);
1646 /* Exit dynarec in case there's a software interrupt.
1647 * exit_flags = !!(status & cause & 0x0300) & status; */
1648 jit_andr(tmp, tmp, status);
1649 jit_andi(tmp, tmp, 0x0300);
1650 jit_nei(tmp, tmp, 0);
1651 jit_andr(tmp, tmp, status);
1652 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1653 LIGHTREC_REG_STATE, tmp);
1655 lightrec_free_reg(reg_cache, status);
1656 lightrec_free_reg(reg_cache, tmp);
1659 static void rec_CP(struct lightrec_cstate *state,
1660 const struct block *block, u16 offset)
1662 union code c = block->opcode_list[offset].c;
1663 jit_state_t *_jit = block->_jit;
1666 jit_note(__FILE__, __LINE__);
1668 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
1671 static void rec_meta_MOV(struct lightrec_cstate *state,
1672 const struct block *block, u16 offset)
1674 struct regcache *reg_cache = state->reg_cache;
1675 union code c = block->opcode_list[offset].c;
1676 jit_state_t *_jit = block->_jit;
1679 _jit_name(block->_jit, __func__);
1680 jit_note(__FILE__, __LINE__);
1682 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
1683 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
1691 lightrec_free_reg(reg_cache, rs);
1692 lightrec_free_reg(reg_cache, rd);
1695 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
1696 const struct block *block,
1699 struct regcache *reg_cache = state->reg_cache;
1700 union code c = block->opcode_list[offset].c;
1701 jit_state_t *_jit = block->_jit;
1704 _jit_name(block->_jit, __func__);
1705 jit_note(__FILE__, __LINE__);
1707 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1708 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1710 if (c.i.op == OP_META_EXTC)
1715 lightrec_free_reg(reg_cache, rs);
1716 lightrec_free_reg(reg_cache, rt);
1719 static const lightrec_rec_func_t rec_standard[64] = {
1720 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
1721 [OP_SPECIAL] = rec_SPECIAL,
1722 [OP_REGIMM] = rec_REGIMM,
1727 [OP_BLEZ] = rec_BLEZ,
1728 [OP_BGTZ] = rec_BGTZ,
1729 [OP_ADDI] = rec_ADDI,
1730 [OP_ADDIU] = rec_ADDIU,
1731 [OP_SLTI] = rec_SLTI,
1732 [OP_SLTIU] = rec_SLTIU,
1733 [OP_ANDI] = rec_ANDI,
1735 [OP_XORI] = rec_XORI,
1751 [OP_LWC2] = rec_LWC2,
1752 [OP_SWC2] = rec_SWC2,
1754 [OP_META_MOV] = rec_meta_MOV,
1755 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
1756 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
1759 static const lightrec_rec_func_t rec_special[64] = {
1760 SET_DEFAULT_ELM(rec_special, unknown_opcode),
1761 [OP_SPECIAL_SLL] = rec_special_SLL,
1762 [OP_SPECIAL_SRL] = rec_special_SRL,
1763 [OP_SPECIAL_SRA] = rec_special_SRA,
1764 [OP_SPECIAL_SLLV] = rec_special_SLLV,
1765 [OP_SPECIAL_SRLV] = rec_special_SRLV,
1766 [OP_SPECIAL_SRAV] = rec_special_SRAV,
1767 [OP_SPECIAL_JR] = rec_special_JR,
1768 [OP_SPECIAL_JALR] = rec_special_JALR,
1769 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
1770 [OP_SPECIAL_BREAK] = rec_special_BREAK,
1771 [OP_SPECIAL_MFHI] = rec_special_MFHI,
1772 [OP_SPECIAL_MTHI] = rec_special_MTHI,
1773 [OP_SPECIAL_MFLO] = rec_special_MFLO,
1774 [OP_SPECIAL_MTLO] = rec_special_MTLO,
1775 [OP_SPECIAL_MULT] = rec_special_MULT,
1776 [OP_SPECIAL_MULTU] = rec_special_MULTU,
1777 [OP_SPECIAL_DIV] = rec_special_DIV,
1778 [OP_SPECIAL_DIVU] = rec_special_DIVU,
1779 [OP_SPECIAL_ADD] = rec_special_ADD,
1780 [OP_SPECIAL_ADDU] = rec_special_ADDU,
1781 [OP_SPECIAL_SUB] = rec_special_SUB,
1782 [OP_SPECIAL_SUBU] = rec_special_SUBU,
1783 [OP_SPECIAL_AND] = rec_special_AND,
1784 [OP_SPECIAL_OR] = rec_special_OR,
1785 [OP_SPECIAL_XOR] = rec_special_XOR,
1786 [OP_SPECIAL_NOR] = rec_special_NOR,
1787 [OP_SPECIAL_SLT] = rec_special_SLT,
1788 [OP_SPECIAL_SLTU] = rec_special_SLTU,
1791 static const lightrec_rec_func_t rec_regimm[64] = {
1792 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
1793 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
1794 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
1795 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
1796 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
1799 static const lightrec_rec_func_t rec_cp0[64] = {
1800 SET_DEFAULT_ELM(rec_cp0, rec_CP),
1801 [OP_CP0_MFC0] = rec_cp0_MFC0,
1802 [OP_CP0_CFC0] = rec_cp0_CFC0,
1803 [OP_CP0_MTC0] = rec_cp0_MTC0,
1804 [OP_CP0_CTC0] = rec_cp0_CTC0,
1805 [OP_CP0_RFE] = rec_cp0_RFE,
1808 static const lightrec_rec_func_t rec_cp2_basic[64] = {
1809 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
1810 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
1811 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
1812 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
1813 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
1816 static void rec_SPECIAL(struct lightrec_cstate *state,
1817 const struct block *block, u16 offset)
1819 union code c = block->opcode_list[offset].c;
1820 lightrec_rec_func_t f = rec_special[c.r.op];
1822 if (!HAS_DEFAULT_ELM && unlikely(!f))
1823 unknown_opcode(state, block, offset);
1825 (*f)(state, block, offset);
1828 static void rec_REGIMM(struct lightrec_cstate *state,
1829 const struct block *block, u16 offset)
1831 union code c = block->opcode_list[offset].c;
1832 lightrec_rec_func_t f = rec_regimm[c.r.rt];
1834 if (!HAS_DEFAULT_ELM && unlikely(!f))
1835 unknown_opcode(state, block, offset);
1837 (*f)(state, block, offset);
1840 static void rec_CP0(struct lightrec_cstate *state,
1841 const struct block *block, u16 offset)
1843 union code c = block->opcode_list[offset].c;
1844 lightrec_rec_func_t f = rec_cp0[c.r.rs];
1846 if (!HAS_DEFAULT_ELM && unlikely(!f))
1847 rec_CP(state, block, offset);
1849 (*f)(state, block, offset);
1852 static void rec_CP2(struct lightrec_cstate *state,
1853 const struct block *block, u16 offset)
1855 union code c = block->opcode_list[offset].c;
1857 if (c.r.op == OP_CP2_BASIC) {
1858 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
1860 if (HAS_DEFAULT_ELM || likely(f)) {
1861 (*f)(state, block, offset);
1866 rec_CP(state, block, offset);
1869 void lightrec_rec_opcode(struct lightrec_cstate *state,
1870 const struct block *block, u16 offset)
1872 struct regcache *reg_cache = state->reg_cache;
1873 struct lightrec_branch_target *target;
1874 const struct opcode *op = &block->opcode_list[offset];
1875 jit_state_t *_jit = block->_jit;
1876 lightrec_rec_func_t f;
1878 if (op->flags & LIGHTREC_SYNC) {
1879 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
1882 lightrec_storeback_regs(reg_cache, _jit);
1883 lightrec_regcache_reset(reg_cache);
1885 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
1886 target = &state->targets[state->nb_targets++];
1887 target->offset = offset;
1888 target->label = jit_indirect();
1891 if (likely(op->opcode)) {
1892 f = rec_standard[op->i.op];
1894 if (!HAS_DEFAULT_ELM && unlikely(!f))
1895 unknown_opcode(state, block, offset);
1897 (*f)(state, block, offset);
1900 if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) {
1901 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true);
1902 pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd));
1904 if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) {
1905 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true);
1906 pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt));
1908 if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) {
1909 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true);
1910 pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt));