1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "disassembler.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
27 pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
28 block->opcode_list[offset].c.opcode,
29 block->pc + (offset << 2));
32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
33 const struct block *block, u16 offset,
34 s8 reg_new_pc, u32 imm, u8 ra_reg,
35 u32 link, bool update_cycles)
37 struct regcache *reg_cache = state->reg_cache;
38 u32 cycles = state->cycles;
39 jit_state_t *_jit = block->_jit;
40 const struct opcode *op = &block->opcode_list[offset],
41 *next = &block->opcode_list[offset + 1];
43 jit_note(__FILE__, __LINE__);
46 /* Update the $ra register */
47 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
48 jit_movi(link_reg, link);
49 lightrec_free_reg(reg_cache, link_reg);
53 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
54 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
56 jit_movi(reg_new_pc, imm);
59 if (has_delay_slot(op->c) &&
60 !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) {
61 cycles += lightrec_cycles_of_opcode(next->c);
63 /* Recompile the delay slot */
65 lightrec_rec_opcode(state, block, offset + 1);
68 /* Store back remaining registers */
69 lightrec_storeback_regs(reg_cache, _jit);
71 jit_movr(JIT_V0, reg_new_pc);
73 if (cycles && update_cycles) {
74 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
75 pr_debug("EOB: %u cycles\n", cycles);
78 if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1)
79 state->branches[state->nb_branches++] = jit_b();
82 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
83 u16 offset, bool after_op)
85 struct regcache *reg_cache = state->reg_cache;
86 jit_state_t *_jit = block->_jit;
87 union code c = block->opcode_list[offset].c;
88 u32 cycles = state->cycles;
91 cycles -= lightrec_cycles_of_opcode(c);
93 lightrec_storeback_regs(reg_cache, _jit);
95 jit_movi(JIT_V0, block->pc + (offset << 2));
96 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
98 state->branches[state->nb_branches++] = jit_b();
101 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
103 struct regcache *reg_cache = state->reg_cache;
104 jit_state_t *_jit = block->_jit;
105 const struct opcode *op = &block->opcode_list[offset];
108 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
109 lightrec_lock_reg(reg_cache, _jit, rs);
114 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
116 u8 rs = get_jr_jalr_reg(state, block, offset);
118 _jit_name(block->_jit, __func__);
119 lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
122 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
124 u8 rs = get_jr_jalr_reg(state, block, offset);
125 union code c = block->opcode_list[offset].c;
127 _jit_name(block->_jit, __func__);
128 lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
129 get_branch_pc(block, offset, 2), true);
132 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
134 union code c = block->opcode_list[offset].c;
136 _jit_name(block->_jit, __func__);
137 lightrec_emit_end_of_block(state, block, offset, -1,
138 (block->pc & 0xf0000000) | (c.j.imm << 2),
142 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
144 union code c = block->opcode_list[offset].c;
146 _jit_name(block->_jit, __func__);
147 lightrec_emit_end_of_block(state, block, offset, -1,
148 (block->pc & 0xf0000000) | (c.j.imm << 2),
149 31, get_branch_pc(block, offset, 2), true);
152 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
153 jit_code_t code, u32 link, bool unconditional, bool bz)
155 struct regcache *reg_cache = state->reg_cache;
156 struct native_register *regs_backup;
157 jit_state_t *_jit = block->_jit;
158 struct lightrec_branch *branch;
159 const struct opcode *op = &block->opcode_list[offset],
160 *next = &block->opcode_list[offset + 1];
163 u32 target_offset, cycles = state->cycles;
164 bool is_forward = (s16)op->i.imm >= -1;
167 jit_note(__FILE__, __LINE__);
169 if (!(op->flags & LIGHTREC_NO_DS))
170 cycles += lightrec_cycles_of_opcode(next->c);
175 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
177 if (!unconditional) {
178 u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT),
179 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
180 _jit, op->i.rt, REG_EXT);
182 /* Generate the branch opcode */
183 addr = jit_new_node_pww(code, NULL, rs, rt);
185 lightrec_free_regs(reg_cache);
186 regs_backup = lightrec_regcache_enter_branch(reg_cache);
189 if (op->flags & LIGHTREC_LOCAL_BRANCH) {
190 if (next && !(op->flags & LIGHTREC_NO_DS)) {
191 /* Recompile the delay slot */
193 lightrec_rec_opcode(state, block, offset + 1);
197 /* Update the $ra register */
198 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
199 jit_movi(link_reg, link);
200 lightrec_free_reg(reg_cache, link_reg);
203 /* Store back remaining registers */
204 lightrec_storeback_regs(reg_cache, _jit);
206 target_offset = offset + 1 + (s16)op->i.imm
207 - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS));
208 pr_debug("Adding local branch to offset 0x%x\n",
210 branch = &state->local_branches[
211 state->nb_local_branches++];
213 branch->target = target_offset;
215 branch->branch = jit_b();
217 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
220 if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) {
221 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
222 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
226 if (!unconditional) {
228 lightrec_regcache_leave_branch(reg_cache, regs_backup);
231 /* Update the $ra register */
232 link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
234 jit_movi(link_reg, (s32)link);
235 lightrec_free_reg(reg_cache, link_reg);
238 if (!(op->flags & LIGHTREC_NO_DS) && next->opcode)
239 lightrec_rec_opcode(state, block, offset + 1);
243 static void rec_BNE(struct lightrec_cstate *state,
244 const struct block *block, u16 offset)
246 union code c = block->opcode_list[offset].c;
248 _jit_name(block->_jit, __func__);
251 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
253 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
256 static void rec_BEQ(struct lightrec_cstate *state,
257 const struct block *block, u16 offset)
259 union code c = block->opcode_list[offset].c;
261 _jit_name(block->_jit, __func__);
264 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
266 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
269 static void rec_BLEZ(struct lightrec_cstate *state,
270 const struct block *block, u16 offset)
272 union code c = block->opcode_list[offset].c;
274 _jit_name(block->_jit, __func__);
275 rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
278 static void rec_BGTZ(struct lightrec_cstate *state,
279 const struct block *block, u16 offset)
281 _jit_name(block->_jit, __func__);
282 rec_b(state, block, offset, jit_code_blei, 0, false, true);
285 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
286 const struct block *block, u16 offset)
288 _jit_name(block->_jit, __func__);
289 rec_b(state, block, offset, jit_code_bgei, 0, false, true);
292 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
293 const struct block *block, u16 offset)
295 _jit_name(block->_jit, __func__);
296 rec_b(state, block, offset, jit_code_bgei,
297 get_branch_pc(block, offset, 2), false, true);
300 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
301 const struct block *block, u16 offset)
303 union code c = block->opcode_list[offset].c;
305 _jit_name(block->_jit, __func__);
306 rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
309 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
310 const struct block *block, u16 offset)
312 const struct opcode *op = &block->opcode_list[offset];
313 _jit_name(block->_jit, __func__);
314 rec_b(state, block, offset, jit_code_blti,
315 get_branch_pc(block, offset, 2),
319 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
320 u16 offset, jit_code_t code, bool slti)
322 struct regcache *reg_cache = state->reg_cache;
323 union code c = block->opcode_list[offset].c;
324 jit_state_t *_jit = block->_jit;
325 u8 rs, rt, out_flags = REG_EXT;
328 out_flags |= REG_ZEXT;
330 jit_note(__FILE__, __LINE__);
331 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
332 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
334 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
336 lightrec_free_reg(reg_cache, rs);
337 lightrec_free_reg(reg_cache, rt);
340 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
341 u16 offset, jit_code_t code, bool out_ext)
343 struct regcache *reg_cache = state->reg_cache;
344 union code c = block->opcode_list[offset].c;
345 jit_state_t *_jit = block->_jit;
348 jit_note(__FILE__, __LINE__);
349 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
350 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
351 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
352 out_ext ? REG_EXT | REG_ZEXT : 0);
354 jit_new_node_www(code, rd, rs, rt);
356 lightrec_free_reg(reg_cache, rs);
357 lightrec_free_reg(reg_cache, rt);
358 lightrec_free_reg(reg_cache, rd);
361 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
362 u16 offset, jit_code_t code)
364 struct regcache *reg_cache = state->reg_cache;
365 union code c = block->opcode_list[offset].c;
366 jit_state_t *_jit = block->_jit;
367 u8 rd, rt, rs, temp, flags = 0;
369 jit_note(__FILE__, __LINE__);
370 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
372 if (code == jit_code_rshr)
374 else if (code == jit_code_rshr_u)
377 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
378 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
380 if (rs != rd && rt != rd) {
381 jit_andi(rd, rs, 0x1f);
382 jit_new_node_www(code, rd, rt, rd);
384 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
385 jit_andi(temp, rs, 0x1f);
386 jit_new_node_www(code, rd, rt, temp);
387 lightrec_free_reg(reg_cache, temp);
390 lightrec_free_reg(reg_cache, rs);
391 lightrec_free_reg(reg_cache, rt);
392 lightrec_free_reg(reg_cache, rd);
395 static void rec_movi(struct lightrec_cstate *state,
396 const struct block *block, u16 offset)
398 struct regcache *reg_cache = state->reg_cache;
399 union code c = block->opcode_list[offset].c;
400 jit_state_t *_jit = block->_jit;
404 if (!(c.i.imm & 0x8000))
407 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
409 jit_movi(rt, (s32)(s16) c.i.imm);
411 lightrec_free_reg(reg_cache, rt);
414 static void rec_ADDIU(struct lightrec_cstate *state,
415 const struct block *block, u16 offset)
417 _jit_name(block->_jit, __func__);
419 if (block->opcode_list[offset].c.i.rs)
420 rec_alu_imm(state, block, offset, jit_code_addi, false);
422 rec_movi(state, block, offset);
425 static void rec_ADDI(struct lightrec_cstate *state,
426 const struct block *block, u16 offset)
428 /* TODO: Handle the exception? */
429 _jit_name(block->_jit, __func__);
430 rec_ADDIU(state, block, offset);
433 static void rec_SLTIU(struct lightrec_cstate *state,
434 const struct block *block, u16 offset)
436 _jit_name(block->_jit, __func__);
437 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
440 static void rec_SLTI(struct lightrec_cstate *state,
441 const struct block *block, u16 offset)
443 _jit_name(block->_jit, __func__);
444 rec_alu_imm(state, block, offset, jit_code_lti, true);
447 static void rec_ANDI(struct lightrec_cstate *state,
448 const struct block *block, u16 offset)
450 struct regcache *reg_cache = state->reg_cache;
451 union code c = block->opcode_list[offset].c;
452 jit_state_t *_jit = block->_jit;
455 _jit_name(block->_jit, __func__);
456 jit_note(__FILE__, __LINE__);
457 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
458 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
461 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
462 * casts to uint8_t / uint16_t. */
465 else if (c.i.imm == 0xffff)
468 jit_andi(rt, rs, (u32)(u16) c.i.imm);
470 lightrec_free_reg(reg_cache, rs);
471 lightrec_free_reg(reg_cache, rt);
474 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
475 u16 offset, jit_code_t code)
477 struct regcache *reg_cache = state->reg_cache;
478 union code c = block->opcode_list[offset].c;
479 jit_state_t *_jit = block->_jit;
482 jit_note(__FILE__, __LINE__);
483 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
484 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
486 flags = lightrec_get_reg_in_flags(reg_cache, rs);
487 lightrec_set_reg_out_flags(reg_cache, rt, flags);
489 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
491 lightrec_free_reg(reg_cache, rs);
492 lightrec_free_reg(reg_cache, rt);
496 static void rec_ORI(struct lightrec_cstate *state,
497 const struct block *block, u16 offset)
499 _jit_name(block->_jit, __func__);
500 rec_alu_or_xor(state, block, offset, jit_code_ori);
503 static void rec_XORI(struct lightrec_cstate *state,
504 const struct block *block, u16 offset)
506 _jit_name(block->_jit, __func__);
507 rec_alu_or_xor(state, block, offset, jit_code_xori);
510 static void rec_LUI(struct lightrec_cstate *state,
511 const struct block *block, u16 offset)
513 struct regcache *reg_cache = state->reg_cache;
514 union code c = block->opcode_list[offset].c;
515 jit_state_t *_jit = block->_jit;
516 u8 rt, flags = REG_EXT;
519 jit_note(__FILE__, __LINE__);
521 if (!(c.i.imm & BIT(15)))
524 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
526 jit_movi(rt, (s32)(c.i.imm << 16));
528 lightrec_free_reg(reg_cache, rt);
531 static void rec_special_ADDU(struct lightrec_cstate *state,
532 const struct block *block, u16 offset)
534 _jit_name(block->_jit, __func__);
535 rec_alu_special(state, block, offset, jit_code_addr, false);
538 static void rec_special_ADD(struct lightrec_cstate *state,
539 const struct block *block, u16 offset)
541 /* TODO: Handle the exception? */
542 _jit_name(block->_jit, __func__);
543 rec_alu_special(state, block, offset, jit_code_addr, false);
546 static void rec_special_SUBU(struct lightrec_cstate *state,
547 const struct block *block, u16 offset)
549 _jit_name(block->_jit, __func__);
550 rec_alu_special(state, block, offset, jit_code_subr, false);
553 static void rec_special_SUB(struct lightrec_cstate *state,
554 const struct block *block, u16 offset)
556 /* TODO: Handle the exception? */
557 _jit_name(block->_jit, __func__);
558 rec_alu_special(state, block, offset, jit_code_subr, false);
561 static void rec_special_AND(struct lightrec_cstate *state,
562 const struct block *block, u16 offset)
564 struct regcache *reg_cache = state->reg_cache;
565 union code c = block->opcode_list[offset].c;
566 jit_state_t *_jit = block->_jit;
567 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
569 _jit_name(block->_jit, __func__);
570 jit_note(__FILE__, __LINE__);
571 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
572 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
573 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
575 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
576 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
578 /* Z(rd) = Z(rs) | Z(rt) */
579 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
581 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
582 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
583 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
584 (REG_EXT & flags_rs & flags_rt))
587 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
589 jit_andr(rd, rs, rt);
591 lightrec_free_reg(reg_cache, rs);
592 lightrec_free_reg(reg_cache, rt);
593 lightrec_free_reg(reg_cache, rd);
596 static void rec_special_or_nor(struct lightrec_cstate *state,
597 const struct block *block, u16 offset, bool nor)
599 struct regcache *reg_cache = state->reg_cache;
600 union code c = block->opcode_list[offset].c;
601 jit_state_t *_jit = block->_jit;
602 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
604 jit_note(__FILE__, __LINE__);
605 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
606 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
607 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
609 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
610 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
612 /* or: Z(rd) = Z(rs) & Z(rt)
615 flags_rd = REG_ZEXT & flags_rs & flags_rt;
617 /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
618 if ((REG_EXT & flags_rs & flags_rt) ||
619 (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
620 (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
623 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
630 lightrec_free_reg(reg_cache, rs);
631 lightrec_free_reg(reg_cache, rt);
632 lightrec_free_reg(reg_cache, rd);
635 static void rec_special_OR(struct lightrec_cstate *state,
636 const struct block *block, u16 offset)
638 _jit_name(block->_jit, __func__);
639 rec_special_or_nor(state, block, offset, false);
642 static void rec_special_NOR(struct lightrec_cstate *state,
643 const struct block *block, u16 offset)
645 _jit_name(block->_jit, __func__);
646 rec_special_or_nor(state, block, offset, true);
649 static void rec_special_XOR(struct lightrec_cstate *state,
650 const struct block *block, u16 offset)
652 struct regcache *reg_cache = state->reg_cache;
653 union code c = block->opcode_list[offset].c;
654 jit_state_t *_jit = block->_jit;
655 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
657 _jit_name(block->_jit, __func__);
659 jit_note(__FILE__, __LINE__);
660 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
661 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
662 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
664 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
665 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
667 /* Z(rd) = Z(rs) & Z(rt) */
668 flags_rd = REG_ZEXT & flags_rs & flags_rt;
670 /* E(rd) = E(rs) & E(rt) */
671 flags_rd |= REG_EXT & flags_rs & flags_rt;
673 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
675 jit_xorr(rd, rs, rt);
677 lightrec_free_reg(reg_cache, rs);
678 lightrec_free_reg(reg_cache, rt);
679 lightrec_free_reg(reg_cache, rd);
682 static void rec_special_SLTU(struct lightrec_cstate *state,
683 const struct block *block, u16 offset)
685 _jit_name(block->_jit, __func__);
686 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
689 static void rec_special_SLT(struct lightrec_cstate *state,
690 const struct block *block, u16 offset)
692 _jit_name(block->_jit, __func__);
693 rec_alu_special(state, block, offset, jit_code_ltr, true);
696 static void rec_special_SLLV(struct lightrec_cstate *state,
697 const struct block *block, u16 offset)
699 _jit_name(block->_jit, __func__);
700 rec_alu_shiftv(state, block, offset, jit_code_lshr);
703 static void rec_special_SRLV(struct lightrec_cstate *state,
704 const struct block *block, u16 offset)
706 _jit_name(block->_jit, __func__);
707 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
710 static void rec_special_SRAV(struct lightrec_cstate *state,
711 const struct block *block, u16 offset)
713 _jit_name(block->_jit, __func__);
714 rec_alu_shiftv(state, block, offset, jit_code_rshr);
717 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
718 u16 offset, jit_code_t code)
720 struct regcache *reg_cache = state->reg_cache;
721 union code c = block->opcode_list[offset].c;
722 jit_state_t *_jit = block->_jit;
723 u8 rd, rt, flags = 0;
725 jit_note(__FILE__, __LINE__);
727 if (code == jit_code_rshi)
729 else if (code == jit_code_rshi_u)
732 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
734 /* Input reg is zero-extended, if we SRL at least by one bit, we know
735 * the output reg will be both zero-extended and sign-extended. */
736 if (code == jit_code_rshi_u && c.r.imm)
738 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
740 jit_new_node_www(code, rd, rt, c.r.imm);
742 lightrec_free_reg(reg_cache, rt);
743 lightrec_free_reg(reg_cache, rd);
746 static void rec_special_SLL(struct lightrec_cstate *state,
747 const struct block *block, u16 offset)
749 _jit_name(block->_jit, __func__);
750 rec_alu_shift(state, block, offset, jit_code_lshi);
753 static void rec_special_SRL(struct lightrec_cstate *state,
754 const struct block *block, u16 offset)
756 _jit_name(block->_jit, __func__);
757 rec_alu_shift(state, block, offset, jit_code_rshi_u);
760 static void rec_special_SRA(struct lightrec_cstate *state,
761 const struct block *block, u16 offset)
763 _jit_name(block->_jit, __func__);
764 rec_alu_shift(state, block, offset, jit_code_rshi);
767 static void rec_alu_mult(struct lightrec_cstate *state,
768 const struct block *block, u16 offset, bool is_signed)
770 struct regcache *reg_cache = state->reg_cache;
771 union code c = block->opcode_list[offset].c;
772 u16 flags = block->opcode_list[offset].flags;
773 u8 reg_lo = get_mult_div_lo(c);
774 u8 reg_hi = get_mult_div_hi(c);
775 jit_state_t *_jit = block->_jit;
776 u8 lo, hi, rs, rt, rflags = 0;
778 jit_note(__FILE__, __LINE__);
785 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
786 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
788 if (!(flags & LIGHTREC_NO_LO))
789 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
790 else if (__WORDSIZE == 32)
791 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
793 if (!(flags & LIGHTREC_NO_HI))
794 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
796 if (__WORDSIZE == 32) {
797 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
798 * operation if the MULT was detected a 32-bit only. */
799 if (!(flags & LIGHTREC_NO_HI)) {
801 jit_qmulr(lo, hi, rs, rt);
803 jit_qmulr_u(lo, hi, rs, rt);
805 jit_mulr(lo, rs, rt);
808 /* On 64-bit systems, do a 64*64->64 bit operation. */
809 if (flags & LIGHTREC_NO_LO) {
810 jit_mulr(hi, rs, rt);
811 jit_rshi(hi, hi, 32);
813 jit_mulr(lo, rs, rt);
815 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
816 if (!(flags & LIGHTREC_NO_HI))
817 jit_rshi(hi, lo, 32);
821 lightrec_free_reg(reg_cache, rs);
822 lightrec_free_reg(reg_cache, rt);
823 if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32)
824 lightrec_free_reg(reg_cache, lo);
825 if (!(flags & LIGHTREC_NO_HI))
826 lightrec_free_reg(reg_cache, hi);
829 static void rec_alu_div(struct lightrec_cstate *state,
830 const struct block *block, u16 offset, bool is_signed)
832 struct regcache *reg_cache = state->reg_cache;
833 union code c = block->opcode_list[offset].c;
834 u16 flags = block->opcode_list[offset].flags;
835 bool no_check = flags & LIGHTREC_NO_DIV_CHECK;
836 u8 reg_lo = get_mult_div_lo(c);
837 u8 reg_hi = get_mult_div_hi(c);
838 jit_state_t *_jit = block->_jit;
839 jit_node_t *branch, *to_end;
840 u8 lo = 0, hi = 0, rs, rt, rflags = 0;
842 jit_note(__FILE__, __LINE__);
849 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
850 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
852 if (!(flags & LIGHTREC_NO_LO))
853 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
855 if (!(flags & LIGHTREC_NO_HI))
856 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
858 /* Jump to special handler if dividing by zero */
860 branch = jit_beqi(rt, 0);
862 if (flags & LIGHTREC_NO_LO) {
864 jit_remr(hi, rs, rt);
866 jit_remr_u(hi, rs, rt);
867 } else if (flags & LIGHTREC_NO_HI) {
869 jit_divr(lo, rs, rt);
871 jit_divr_u(lo, rs, rt);
874 jit_qdivr(lo, hi, rs, rt);
876 jit_qdivr_u(lo, hi, rs, rt);
880 /* Jump above the div-by-zero handler */
885 if (!(flags & LIGHTREC_NO_LO)) {
891 jit_movi(lo, 0xffffffff);
895 if (!(flags & LIGHTREC_NO_HI))
901 lightrec_free_reg(reg_cache, rs);
902 lightrec_free_reg(reg_cache, rt);
904 if (!(flags & LIGHTREC_NO_LO))
905 lightrec_free_reg(reg_cache, lo);
907 if (!(flags & LIGHTREC_NO_HI))
908 lightrec_free_reg(reg_cache, hi);
911 static void rec_special_MULT(struct lightrec_cstate *state,
912 const struct block *block, u16 offset)
914 _jit_name(block->_jit, __func__);
915 rec_alu_mult(state, block, offset, true);
918 static void rec_special_MULTU(struct lightrec_cstate *state,
919 const struct block *block, u16 offset)
921 _jit_name(block->_jit, __func__);
922 rec_alu_mult(state, block, offset, false);
925 static void rec_special_DIV(struct lightrec_cstate *state,
926 const struct block *block, u16 offset)
928 _jit_name(block->_jit, __func__);
929 rec_alu_div(state, block, offset, true);
932 static void rec_special_DIVU(struct lightrec_cstate *state,
933 const struct block *block, u16 offset)
935 _jit_name(block->_jit, __func__);
936 rec_alu_div(state, block, offset, false);
939 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
940 const struct block *block, u8 dst, u8 src)
942 struct regcache *reg_cache = state->reg_cache;
943 jit_state_t *_jit = block->_jit;
945 jit_note(__FILE__, __LINE__);
946 src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
947 dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
949 jit_extr_i(dst, src);
951 lightrec_free_reg(reg_cache, src);
952 lightrec_free_reg(reg_cache, dst);
955 static void rec_special_MFHI(struct lightrec_cstate *state,
956 const struct block *block, u16 offset)
958 union code c = block->opcode_list[offset].c;
960 _jit_name(block->_jit, __func__);
961 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
964 static void rec_special_MTHI(struct lightrec_cstate *state,
965 const struct block *block, u16 offset)
967 union code c = block->opcode_list[offset].c;
969 _jit_name(block->_jit, __func__);
970 rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
973 static void rec_special_MFLO(struct lightrec_cstate *state,
974 const struct block *block, u16 offset)
976 union code c = block->opcode_list[offset].c;
978 _jit_name(block->_jit, __func__);
979 rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
982 static void rec_special_MTLO(struct lightrec_cstate *state,
983 const struct block *block, u16 offset)
985 union code c = block->opcode_list[offset].c;
987 _jit_name(block->_jit, __func__);
988 rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
991 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
992 u32 arg, bool with_arg, enum c_wrappers wrapper)
994 struct regcache *reg_cache = state->reg_cache;
995 jit_state_t *_jit = block->_jit;
998 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
999 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1000 offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1003 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1004 jit_movi(tmp2, arg);
1006 jit_stxi_i(offsetof(struct lightrec_state, c_wrapper_arg),
1007 LIGHTREC_REG_STATE, tmp2);
1009 lightrec_free_reg(reg_cache, tmp2);
1012 lightrec_regcache_mark_live(reg_cache, _jit);
1015 lightrec_free_reg(reg_cache, tmp);
1016 lightrec_regcache_mark_live(reg_cache, _jit);
1019 static void rec_io(struct lightrec_cstate *state,
1020 const struct block *block, u16 offset,
1021 bool load_rt, bool read_rt)
1023 struct regcache *reg_cache = state->reg_cache;
1024 jit_state_t *_jit = block->_jit;
1025 union code c = block->opcode_list[offset].c;
1026 u16 flags = block->opcode_list[offset].flags;
1027 bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1030 jit_note(__FILE__, __LINE__);
1032 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1034 if (read_rt && likely(c.i.rt))
1035 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1037 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1040 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1042 lut_entry = lightrec_get_lut_entry(block);
1043 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1044 true, C_WRAPPER_RW_GENERIC);
1048 static u32 rec_ram_mask(struct lightrec_state *state)
1050 return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1053 static void rec_store_memory(struct lightrec_cstate *cstate,
1054 const struct block *block,
1055 u16 offset, jit_code_t code,
1056 jit_code_t swap_code,
1057 uintptr_t addr_offset, u32 addr_mask,
1060 const struct lightrec_state *state = cstate->state;
1061 struct regcache *reg_cache = cstate->reg_cache;
1062 struct opcode *op = &block->opcode_list[offset];
1063 jit_state_t *_jit = block->_jit;
1064 union code c = op->c;
1065 u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1066 s16 imm = (s16)c.i.imm;
1067 s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1068 s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1069 bool no_mask = op->flags & LIGHTREC_NO_MASK;
1070 bool add_imm = c.i.imm &&
1071 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1072 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1073 bool need_tmp = !no_mask || addr_offset || add_imm;
1074 bool need_tmp2 = addr_offset || invalidate;
1076 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1077 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1079 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1084 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1092 jit_andi(tmp, addr_reg, addr_mask);
1097 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1100 jit_addi(tmp2, addr_reg, addr_offset);
1103 addr_reg2 = addr_reg;
1106 if (is_big_endian() && swap_code && c.i.rt) {
1107 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1109 jit_new_node_ww(swap_code, tmp3, rt);
1110 jit_new_node_www(code, imm, addr_reg2, tmp3);
1112 lightrec_free_reg(reg_cache, tmp3);
1114 jit_new_node_www(code, imm, addr_reg2, rt);
1117 lightrec_free_reg(reg_cache, rt);
1120 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1122 if (c.i.op != OP_SW) {
1123 jit_andi(tmp2, addr_reg, ~3);
1127 if (!lut_is_32bit(state)) {
1128 jit_lshi(tmp2, addr_reg, 1);
1132 if (addr_reg == rs && c.i.rs == 0) {
1133 addr_reg = LIGHTREC_REG_STATE;
1135 jit_addr(tmp2, addr_reg, LIGHTREC_REG_STATE);
1139 if (lut_is_32bit(state))
1140 jit_stxi_i(lut_offt, addr_reg, tmp3);
1142 jit_stxi(lut_offt, addr_reg, tmp3);
1144 lightrec_free_reg(reg_cache, tmp3);
1148 lightrec_free_reg(reg_cache, tmp2);
1150 lightrec_free_reg(reg_cache, tmp);
1151 lightrec_free_reg(reg_cache, rs);
1154 static void rec_store_ram(struct lightrec_cstate *cstate,
1155 const struct block *block,
1156 u16 offset, jit_code_t code,
1157 jit_code_t swap_code, bool invalidate)
1159 struct lightrec_state *state = cstate->state;
1161 _jit_note(block->_jit, __FILE__, __LINE__);
1163 return rec_store_memory(cstate, block, offset, code, swap_code,
1164 state->offset_ram, rec_ram_mask(state),
1168 static void rec_store_scratch(struct lightrec_cstate *cstate,
1169 const struct block *block, u16 offset,
1170 jit_code_t code, jit_code_t swap_code)
1172 _jit_note(block->_jit, __FILE__, __LINE__);
1174 return rec_store_memory(cstate, block, offset, code, swap_code,
1175 cstate->state->offset_scratch,
1179 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1180 const struct block *block,
1181 u16 offset, jit_code_t code,
1182 jit_code_t swap_code)
1184 struct lightrec_state *state = cstate->state;
1185 struct regcache *reg_cache = cstate->reg_cache;
1186 union code c = block->opcode_list[offset].c;
1187 jit_state_t *_jit = block->_jit;
1188 jit_node_t *to_not_ram, *to_end;
1189 u8 tmp, tmp2, rs, rt;
1192 jit_note(__FILE__, __LINE__);
1193 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1194 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1196 if (state->offset_ram || state->offset_scratch)
1197 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1199 /* Convert to KUNSEG and avoid RAM mirrors */
1200 if (state->mirrors_mapped) {
1202 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1203 } else if (c.i.imm) {
1205 jit_addi(tmp, rs, (s16)c.i.imm);
1206 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1209 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1212 lightrec_free_reg(reg_cache, rs);
1214 if (state->offset_ram != state->offset_scratch) {
1215 to_not_ram = jit_bmsi(tmp, BIT(28));
1217 jit_movi(tmp2, state->offset_ram);
1220 jit_patch(to_not_ram);
1222 jit_movi(tmp2, state->offset_scratch);
1224 } else if (state->offset_ram) {
1225 jit_movi(tmp2, state->offset_ram);
1228 if (state->offset_ram || state->offset_scratch) {
1229 jit_addr(tmp, tmp, tmp2);
1230 lightrec_free_reg(reg_cache, tmp2);
1233 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1235 if (is_big_endian() && swap_code && c.i.rt) {
1236 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1238 jit_new_node_ww(swap_code, tmp2, rt);
1239 jit_new_node_www(code, imm, tmp, tmp2);
1241 lightrec_free_reg(reg_cache, tmp2);
1243 jit_new_node_www(code, imm, tmp, rt);
1246 lightrec_free_reg(reg_cache, rt);
1247 lightrec_free_reg(reg_cache, tmp);
1250 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1251 u16 offset, jit_code_t code, jit_code_t swap_code)
1253 struct lightrec_state *state = cstate->state;
1254 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1255 struct regcache *reg_cache = cstate->reg_cache;
1256 union code c = block->opcode_list[offset].c;
1257 jit_state_t *_jit = block->_jit;
1258 jit_node_t *to_not_ram, *to_end;
1259 u8 tmp, tmp2, tmp3, rs, rt;
1261 jit_note(__FILE__, __LINE__);
1263 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1264 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1265 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1267 /* Convert to KUNSEG and avoid RAM mirrors */
1269 jit_addi(tmp2, rs, (s16)c.i.imm);
1270 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1272 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1275 lightrec_free_reg(reg_cache, rs);
1276 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1278 to_not_ram = jit_bgti(tmp2, ram_size);
1280 /* Compute the offset to the code LUT */
1281 jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1282 if (!lut_is_32bit(state))
1283 jit_lshi(tmp, tmp, 1);
1284 jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1286 /* Write NULL to the code LUT to invalidate any block that's there */
1287 if (lut_is_32bit(state))
1288 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1290 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1292 if (state->offset_ram != state->offset_scratch) {
1293 jit_movi(tmp, state->offset_ram);
1298 jit_patch(to_not_ram);
1300 if (state->offset_ram || state->offset_scratch)
1301 jit_movi(tmp, state->offset_scratch);
1303 if (state->offset_ram != state->offset_scratch)
1306 if (state->offset_ram || state->offset_scratch)
1307 jit_addr(tmp2, tmp2, tmp);
1309 lightrec_free_reg(reg_cache, tmp);
1310 lightrec_free_reg(reg_cache, tmp3);
1312 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1314 if (is_big_endian() && swap_code && c.i.rt) {
1315 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1317 jit_new_node_ww(swap_code, tmp, rt);
1318 jit_new_node_www(code, 0, tmp2, tmp);
1320 lightrec_free_reg(reg_cache, tmp);
1322 jit_new_node_www(code, 0, tmp2, rt);
1325 lightrec_free_reg(reg_cache, rt);
1326 lightrec_free_reg(reg_cache, tmp2);
1329 static void rec_store(struct lightrec_cstate *state,
1330 const struct block *block, u16 offset,
1331 jit_code_t code, jit_code_t swap_code)
1333 u16 flags = block->opcode_list[offset].flags;
1334 bool no_invalidate = (flags & LIGHTREC_NO_INVALIDATE) ||
1335 state->state->invalidate_from_dma_only;
1337 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1338 case LIGHTREC_IO_RAM:
1339 rec_store_ram(state, block, offset, code,
1340 swap_code, !no_invalidate);
1342 case LIGHTREC_IO_SCRATCH:
1343 rec_store_scratch(state, block, offset, code, swap_code);
1345 case LIGHTREC_IO_DIRECT:
1346 if (no_invalidate) {
1347 rec_store_direct_no_invalidate(state, block, offset,
1350 rec_store_direct(state, block, offset, code, swap_code);
1354 rec_io(state, block, offset, true, false);
1359 static void rec_SB(struct lightrec_cstate *state,
1360 const struct block *block, u16 offset)
1362 _jit_name(block->_jit, __func__);
1363 rec_store(state, block, offset, jit_code_stxi_c, 0);
1366 static void rec_SH(struct lightrec_cstate *state,
1367 const struct block *block, u16 offset)
1369 _jit_name(block->_jit, __func__);
1370 rec_store(state, block, offset,
1371 jit_code_stxi_s, jit_code_bswapr_us);
1374 static void rec_SW(struct lightrec_cstate *state,
1375 const struct block *block, u16 offset)
1378 _jit_name(block->_jit, __func__);
1379 rec_store(state, block, offset,
1380 jit_code_stxi_i, jit_code_bswapr_ui);
1383 static void rec_SWL(struct lightrec_cstate *state,
1384 const struct block *block, u16 offset)
1386 _jit_name(block->_jit, __func__);
1387 rec_io(state, block, offset, true, false);
1390 static void rec_SWR(struct lightrec_cstate *state,
1391 const struct block *block, u16 offset)
1393 _jit_name(block->_jit, __func__);
1394 rec_io(state, block, offset, true, false);
1397 static void rec_SWC2(struct lightrec_cstate *state,
1398 const struct block *block, u16 offset)
1400 _jit_name(block->_jit, __func__);
1401 rec_io(state, block, offset, false, false);
1404 static void rec_load_memory(struct lightrec_cstate *cstate,
1405 const struct block *block, u16 offset,
1406 jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1407 uintptr_t addr_offset, u32 addr_mask)
1409 struct regcache *reg_cache = cstate->reg_cache;
1410 struct opcode *op = &block->opcode_list[offset];
1411 jit_state_t *_jit = block->_jit;
1412 u8 rs, rt, addr_reg, flags = REG_EXT;
1413 bool no_mask = op->flags & LIGHTREC_NO_MASK;
1414 union code c = op->c;
1423 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1424 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1426 if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1427 jit_addi(rt, rs, (s16)c.i.imm);
1436 jit_andi(rt, addr_reg, addr_mask);
1441 jit_addi(rt, addr_reg, addr_offset);
1445 jit_new_node_www(code, rt, addr_reg, imm);
1447 if (is_big_endian() && swap_code) {
1448 jit_new_node_ww(swap_code, rt, rt);
1450 if (c.i.op == OP_LH)
1452 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1456 lightrec_free_reg(reg_cache, rs);
1457 lightrec_free_reg(reg_cache, rt);
1460 static void rec_load_ram(struct lightrec_cstate *cstate,
1461 const struct block *block, u16 offset,
1462 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1464 _jit_note(block->_jit, __FILE__, __LINE__);
1466 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1467 cstate->state->offset_ram, rec_ram_mask(cstate->state));
1470 static void rec_load_bios(struct lightrec_cstate *cstate,
1471 const struct block *block, u16 offset,
1472 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1474 _jit_note(block->_jit, __FILE__, __LINE__);
1476 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1477 cstate->state->offset_bios, 0x1fffffff);
1480 static void rec_load_scratch(struct lightrec_cstate *cstate,
1481 const struct block *block, u16 offset,
1482 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1484 _jit_note(block->_jit, __FILE__, __LINE__);
1486 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1487 cstate->state->offset_scratch, 0x1fffffff);
1490 static void rec_load_direct(struct lightrec_cstate *cstate,
1491 const struct block *block, u16 offset,
1492 jit_code_t code, jit_code_t swap_code,
1495 struct lightrec_state *state = cstate->state;
1496 struct regcache *reg_cache = cstate->reg_cache;
1497 union code c = block->opcode_list[offset].c;
1498 jit_state_t *_jit = block->_jit;
1499 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1500 u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1509 jit_note(__FILE__, __LINE__);
1510 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1511 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1513 if ((state->offset_ram == state->offset_bios &&
1514 state->offset_ram == state->offset_scratch &&
1515 state->mirrors_mapped) || !c.i.imm) {
1519 jit_addi(rt, rs, (s16)c.i.imm);
1523 if (c.i.rs != c.i.rt)
1524 lightrec_free_reg(reg_cache, rs);
1527 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1529 if (state->offset_ram == state->offset_bios &&
1530 state->offset_ram == state->offset_scratch) {
1531 if (!state->mirrors_mapped) {
1532 jit_andi(tmp, addr_reg, BIT(28));
1533 jit_rshi_u(tmp, tmp, 28 - 22);
1534 jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1535 jit_andr(rt, addr_reg, tmp);
1537 jit_andi(rt, addr_reg, 0x1fffffff);
1540 if (state->offset_ram)
1541 jit_movi(tmp, state->offset_ram);
1543 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1545 /* Convert to KUNSEG and avoid RAM mirrors */
1546 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1548 if (state->offset_ram)
1549 jit_movi(tmp, state->offset_ram);
1553 jit_patch(to_not_ram);
1555 if (state->offset_bios != state->offset_scratch)
1556 to_not_bios = jit_bmci(addr_reg, BIT(22));
1558 /* Convert to KUNSEG */
1559 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1561 jit_movi(tmp, state->offset_bios);
1563 if (state->offset_bios != state->offset_scratch) {
1566 jit_patch(to_not_bios);
1568 /* Convert to KUNSEG */
1569 jit_andi(rt, addr_reg, 0x1f800fff);
1571 if (state->offset_scratch)
1572 jit_movi(tmp, state->offset_scratch);
1580 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1581 jit_addr(rt, rt, tmp);
1583 jit_new_node_www(code, rt, rt, imm);
1585 if (is_big_endian() && swap_code) {
1586 jit_new_node_ww(swap_code, rt, rt);
1588 if (c.i.op == OP_LH)
1590 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1594 lightrec_free_reg(reg_cache, addr_reg);
1595 lightrec_free_reg(reg_cache, rt);
1596 lightrec_free_reg(reg_cache, tmp);
1599 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1600 u16 offset, jit_code_t code, jit_code_t swap_code,
1603 u16 flags = block->opcode_list[offset].flags;
1605 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1606 case LIGHTREC_IO_RAM:
1607 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1609 case LIGHTREC_IO_BIOS:
1610 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1612 case LIGHTREC_IO_SCRATCH:
1613 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1615 case LIGHTREC_IO_DIRECT:
1616 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1619 rec_io(state, block, offset, false, true);
1624 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1626 _jit_name(block->_jit, __func__);
1627 rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1630 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1632 _jit_name(block->_jit, __func__);
1633 rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1636 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1638 _jit_name(block->_jit, __func__);
1639 rec_load(state, block, offset, jit_code_ldxi_s, jit_code_bswapr_us, false);
1642 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1644 _jit_name(block->_jit, __func__);
1645 rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1648 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1650 _jit_name(block->_jit, __func__);
1651 rec_io(state, block, offset, true, true);
1654 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1656 _jit_name(block->_jit, __func__);
1657 rec_io(state, block, offset, true, true);
1660 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1662 _jit_name(block->_jit, __func__);
1663 rec_load(state, block, offset, jit_code_ldxi_i, jit_code_bswapr_ui, false);
1666 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1668 _jit_name(block->_jit, __func__);
1669 rec_io(state, block, offset, false, false);
1672 static void rec_break_syscall(struct lightrec_cstate *state,
1673 const struct block *block, u16 offset, bool is_break)
1675 _jit_note(block->_jit, __FILE__, __LINE__);
1678 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1680 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1682 /* TODO: the return address should be "pc - 4" if we're a delay slot */
1683 lightrec_emit_end_of_block(state, block, offset, -1,
1684 get_ds_pc(block, offset, 0),
1688 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1689 const struct block *block, u16 offset)
1691 _jit_name(block->_jit, __func__);
1692 rec_break_syscall(state, block, offset, false);
1695 static void rec_special_BREAK(struct lightrec_cstate *state,
1696 const struct block *block, u16 offset)
1698 _jit_name(block->_jit, __func__);
1699 rec_break_syscall(state, block, offset, true);
1702 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1704 struct regcache *reg_cache = state->reg_cache;
1705 union code c = block->opcode_list[offset].c;
1706 jit_state_t *_jit = block->_jit;
1708 jit_note(__FILE__, __LINE__);
1709 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1710 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1712 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1714 if (c.i.op == OP_CP0 &&
1715 !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1716 (c.r.rd == 12 || c.r.rd == 13))
1717 lightrec_emit_end_of_block(state, block, offset, -1,
1718 get_ds_pc(block, offset, 1),
1723 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1725 struct regcache *reg_cache = state->reg_cache;
1726 union code c = block->opcode_list[offset].c;
1727 jit_state_t *_jit = block->_jit;
1730 jit_note(__FILE__, __LINE__);
1732 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1734 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1735 offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1737 lightrec_free_reg(reg_cache, rt);
1740 static bool block_in_bios(const struct lightrec_cstate *state,
1741 const struct block *block)
1743 const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1744 u32 pc = kunseg(block->pc);
1746 return pc >= bios->pc && pc < bios->pc + bios->length;
1750 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1752 struct regcache *reg_cache = state->reg_cache;
1753 const union code c = block->opcode_list[offset].c;
1754 jit_state_t *_jit = block->_jit;
1755 u8 rt, tmp = 0, tmp2, status;
1757 jit_note(__FILE__, __LINE__);
1765 /* Those registers are read-only */
1771 if (block_in_bios(state, block) && c.r.rd == 12) {
1772 /* If we are running code from the BIOS, handle writes to the
1773 * Status register in C. BIOS code may toggle bit 16 which will
1774 * map/unmap the RAM, while game code cannot do that. */
1775 rec_mtc(state, block, offset);
1779 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1782 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1783 LIGHTREC_REG_STATE, rt);
1786 if (c.r.rd == 12 || c.r.rd == 13) {
1787 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1788 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1789 offsetof(struct lightrec_state, regs.cp0[13]));
1791 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1796 } else if (c.r.rd == 13) {
1797 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1798 jit_andi(tmp2, rt, 0x0300);
1799 jit_ori(tmp, tmp, 0x0300);
1800 jit_xori(tmp, tmp, 0x0300);
1801 jit_orr(tmp, tmp, tmp2);
1802 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1803 offsetof(struct lightrec_state, regs.cp0[12]));
1804 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1805 LIGHTREC_REG_STATE, tmp);
1809 if (c.r.rd == 12 || c.r.rd == 13) {
1810 /* Exit dynarec in case there's a software interrupt.
1811 * exit_flags = !!(status & tmp & 0x0300) & status; */
1812 jit_andr(tmp, tmp, status);
1813 jit_andi(tmp, tmp, 0x0300);
1814 jit_nei(tmp, tmp, 0);
1815 jit_andr(tmp, tmp, status);
1819 /* Exit dynarec in case we unmask a hardware interrupt.
1820 * exit_flags = !(~status & 0x401) */
1822 jit_comr(tmp2, status);
1823 jit_andi(tmp2, tmp2, 0x401);
1824 jit_eqi(tmp2, tmp2, 0);
1825 jit_orr(tmp, tmp, tmp2);
1828 if (c.r.rd == 12 || c.r.rd == 13) {
1829 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1830 LIGHTREC_REG_STATE, tmp);
1832 lightrec_free_reg(reg_cache, tmp);
1833 lightrec_free_reg(reg_cache, tmp2);
1836 lightrec_free_reg(reg_cache, rt);
1838 if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1839 (c.r.rd == 12 || c.r.rd == 13))
1840 lightrec_emit_eob(state, block, offset + 1, true);
1843 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1844 const struct block *block, u16 offset)
1846 _jit_name(block->_jit, __func__);
1847 rec_mfc0(state, block, offset);
1850 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1851 const struct block *block, u16 offset)
1853 _jit_name(block->_jit, __func__);
1854 rec_mfc0(state, block, offset);
1857 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1858 const struct block *block, u16 offset)
1860 _jit_name(block->_jit, __func__);
1861 rec_mtc0(state, block, offset);
1864 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1865 const struct block *block, u16 offset)
1867 _jit_name(block->_jit, __func__);
1868 rec_mtc0(state, block, offset);
1871 static unsigned int cp2d_i_offset(u8 reg)
1873 return offsetof(struct lightrec_state, regs.cp2d[reg]);
1876 static unsigned int cp2d_s_offset(u8 reg)
1878 return cp2d_i_offset(reg) + is_big_endian() * 2;
1881 static unsigned int cp2c_i_offset(u8 reg)
1883 return offsetof(struct lightrec_state, regs.cp2c[reg]);
1886 static unsigned int cp2c_s_offset(u8 reg)
1888 return cp2c_i_offset(reg) + is_big_endian() * 2;
1891 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1892 const struct block *block, u16 offset)
1894 struct regcache *reg_cache = state->reg_cache;
1895 const union code c = block->opcode_list[offset].c;
1896 jit_state_t *_jit = block->_jit;
1897 const u32 zext_regs = 0x300f0080;
1898 u8 rt, tmp, tmp2, tmp3, out, flags;
1899 u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
1902 _jit_name(block->_jit, __func__);
1904 flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
1905 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
1915 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1922 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1926 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1927 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1928 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1930 for (i = 0; i < 3; i++) {
1931 out = i == 0 ? rt : tmp;
1933 jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
1934 jit_movi(tmp2, 0x1f);
1935 jit_rshi(out, tmp, 7);
1937 jit_ltr(tmp3, tmp2, out);
1938 jit_movnr(out, tmp2, tmp3);
1940 jit_gei(tmp2, out, 0);
1941 jit_movzr(out, tmp2, tmp2);
1944 jit_lshi(tmp, tmp, 5 * i);
1945 jit_orr(rt, rt, tmp);
1950 lightrec_free_reg(reg_cache, tmp);
1951 lightrec_free_reg(reg_cache, tmp2);
1952 lightrec_free_reg(reg_cache, tmp3);
1955 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
1959 lightrec_free_reg(reg_cache, rt);
1962 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
1963 const struct block *block, u16 offset)
1965 struct regcache *reg_cache = state->reg_cache;
1966 const union code c = block->opcode_list[offset].c;
1967 jit_state_t *_jit = block->_jit;
1970 _jit_name(block->_jit, __func__);
1980 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
1981 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
1984 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
1985 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
1989 lightrec_free_reg(reg_cache, rt);
1992 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
1993 const struct block *block, u16 offset)
1995 struct regcache *reg_cache = state->reg_cache;
1996 const union code c = block->opcode_list[offset].c;
1997 jit_state_t *_jit = block->_jit;
1998 jit_node_t *loop, *to_loop;
1999 u8 rt, tmp, tmp2, flags = 0;
2001 _jit_name(block->_jit, __func__);
2009 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2013 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2014 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2016 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2017 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2019 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2020 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2021 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2023 lightrec_free_reg(reg_cache, tmp);
2024 lightrec_free_reg(reg_cache, tmp2);
2027 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2029 jit_lshi(tmp, rt, 7);
2030 jit_andi(tmp, tmp, 0xf80);
2031 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2033 jit_lshi(tmp, rt, 2);
2034 jit_andi(tmp, tmp, 0xf80);
2035 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2037 jit_rshi(tmp, rt, 3);
2038 jit_andi(tmp, tmp, 0xf80);
2039 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2041 lightrec_free_reg(reg_cache, tmp);
2044 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2045 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2047 /* if (rt < 0) rt = ~rt; */
2048 jit_rshi(tmp, rt, 31);
2049 jit_xorr(tmp, rt, tmp);
2051 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2052 jit_lshi(tmp, tmp, 1);
2055 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2057 jit_subi(tmp2, tmp2, 1);
2058 jit_rshi_u(tmp, tmp, 1);
2059 to_loop = jit_bnei(tmp, 0);
2061 jit_patch_at(to_loop, loop);
2063 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2064 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2066 lightrec_free_reg(reg_cache, tmp);
2067 lightrec_free_reg(reg_cache, tmp2);
2070 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2074 lightrec_free_reg(reg_cache, rt);
2077 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2078 const struct block *block, u16 offset)
2080 struct regcache *reg_cache = state->reg_cache;
2081 const union code c = block->opcode_list[offset].c;
2082 jit_state_t *_jit = block->_jit;
2085 _jit_name(block->_jit, __func__);
2087 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2097 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2100 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2101 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2103 jit_andi(tmp, rt, 0x7f87e000);
2104 jit_nei(tmp, tmp, 0);
2105 jit_lshi(tmp, tmp, 31);
2107 jit_andi(tmp2, rt, 0x7ffff000);
2108 jit_orr(tmp, tmp2, tmp);
2110 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2112 lightrec_free_reg(reg_cache, tmp);
2113 lightrec_free_reg(reg_cache, tmp2);
2117 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2120 lightrec_free_reg(reg_cache, rt);
2123 static void rec_cp0_RFE(struct lightrec_cstate *state,
2124 const struct block *block, u16 offset)
2126 struct regcache *reg_cache = state->reg_cache;
2127 jit_state_t *_jit = block->_jit;
2131 jit_note(__FILE__, __LINE__);
2133 status = lightrec_alloc_reg_temp(reg_cache, _jit);
2134 jit_ldxi_i(status, LIGHTREC_REG_STATE,
2135 offsetof(struct lightrec_state, regs.cp0[12]));
2137 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2139 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2140 jit_rshi(tmp, status, 2);
2141 jit_andi(tmp, tmp, 0xf);
2142 jit_andi(status, status, ~0xful);
2143 jit_orr(status, status, tmp);
2145 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2146 offsetof(struct lightrec_state, regs.cp0[13]));
2147 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2148 LIGHTREC_REG_STATE, status);
2150 /* Exit dynarec in case there's a software interrupt.
2151 * exit_flags = !!(status & cause & 0x0300) & status; */
2152 jit_andr(tmp, tmp, status);
2153 jit_andi(tmp, tmp, 0x0300);
2154 jit_nei(tmp, tmp, 0);
2155 jit_andr(tmp, tmp, status);
2156 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2157 LIGHTREC_REG_STATE, tmp);
2159 lightrec_free_reg(reg_cache, status);
2160 lightrec_free_reg(reg_cache, tmp);
2163 static void rec_CP(struct lightrec_cstate *state,
2164 const struct block *block, u16 offset)
2166 union code c = block->opcode_list[offset].c;
2167 jit_state_t *_jit = block->_jit;
2170 jit_note(__FILE__, __LINE__);
2172 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
2175 static void rec_meta_MOV(struct lightrec_cstate *state,
2176 const struct block *block, u16 offset)
2178 struct regcache *reg_cache = state->reg_cache;
2179 union code c = block->opcode_list[offset].c;
2180 jit_state_t *_jit = block->_jit;
2183 _jit_name(block->_jit, __func__);
2184 jit_note(__FILE__, __LINE__);
2186 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2187 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2195 lightrec_free_reg(reg_cache, rs);
2196 lightrec_free_reg(reg_cache, rd);
2199 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2200 const struct block *block,
2203 struct regcache *reg_cache = state->reg_cache;
2204 union code c = block->opcode_list[offset].c;
2205 jit_state_t *_jit = block->_jit;
2208 _jit_name(block->_jit, __func__);
2209 jit_note(__FILE__, __LINE__);
2211 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2212 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2214 if (c.i.op == OP_META_EXTC)
2219 lightrec_free_reg(reg_cache, rs);
2220 lightrec_free_reg(reg_cache, rt);
2223 static const lightrec_rec_func_t rec_standard[64] = {
2224 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2225 [OP_SPECIAL] = rec_SPECIAL,
2226 [OP_REGIMM] = rec_REGIMM,
2231 [OP_BLEZ] = rec_BLEZ,
2232 [OP_BGTZ] = rec_BGTZ,
2233 [OP_ADDI] = rec_ADDI,
2234 [OP_ADDIU] = rec_ADDIU,
2235 [OP_SLTI] = rec_SLTI,
2236 [OP_SLTIU] = rec_SLTIU,
2237 [OP_ANDI] = rec_ANDI,
2239 [OP_XORI] = rec_XORI,
2255 [OP_LWC2] = rec_LWC2,
2256 [OP_SWC2] = rec_SWC2,
2258 [OP_META_MOV] = rec_meta_MOV,
2259 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
2260 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
2263 static const lightrec_rec_func_t rec_special[64] = {
2264 SET_DEFAULT_ELM(rec_special, unknown_opcode),
2265 [OP_SPECIAL_SLL] = rec_special_SLL,
2266 [OP_SPECIAL_SRL] = rec_special_SRL,
2267 [OP_SPECIAL_SRA] = rec_special_SRA,
2268 [OP_SPECIAL_SLLV] = rec_special_SLLV,
2269 [OP_SPECIAL_SRLV] = rec_special_SRLV,
2270 [OP_SPECIAL_SRAV] = rec_special_SRAV,
2271 [OP_SPECIAL_JR] = rec_special_JR,
2272 [OP_SPECIAL_JALR] = rec_special_JALR,
2273 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
2274 [OP_SPECIAL_BREAK] = rec_special_BREAK,
2275 [OP_SPECIAL_MFHI] = rec_special_MFHI,
2276 [OP_SPECIAL_MTHI] = rec_special_MTHI,
2277 [OP_SPECIAL_MFLO] = rec_special_MFLO,
2278 [OP_SPECIAL_MTLO] = rec_special_MTLO,
2279 [OP_SPECIAL_MULT] = rec_special_MULT,
2280 [OP_SPECIAL_MULTU] = rec_special_MULTU,
2281 [OP_SPECIAL_DIV] = rec_special_DIV,
2282 [OP_SPECIAL_DIVU] = rec_special_DIVU,
2283 [OP_SPECIAL_ADD] = rec_special_ADD,
2284 [OP_SPECIAL_ADDU] = rec_special_ADDU,
2285 [OP_SPECIAL_SUB] = rec_special_SUB,
2286 [OP_SPECIAL_SUBU] = rec_special_SUBU,
2287 [OP_SPECIAL_AND] = rec_special_AND,
2288 [OP_SPECIAL_OR] = rec_special_OR,
2289 [OP_SPECIAL_XOR] = rec_special_XOR,
2290 [OP_SPECIAL_NOR] = rec_special_NOR,
2291 [OP_SPECIAL_SLT] = rec_special_SLT,
2292 [OP_SPECIAL_SLTU] = rec_special_SLTU,
2295 static const lightrec_rec_func_t rec_regimm[64] = {
2296 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2297 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
2298 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
2299 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
2300 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
2303 static const lightrec_rec_func_t rec_cp0[64] = {
2304 SET_DEFAULT_ELM(rec_cp0, rec_CP),
2305 [OP_CP0_MFC0] = rec_cp0_MFC0,
2306 [OP_CP0_CFC0] = rec_cp0_CFC0,
2307 [OP_CP0_MTC0] = rec_cp0_MTC0,
2308 [OP_CP0_CTC0] = rec_cp0_CTC0,
2309 [OP_CP0_RFE] = rec_cp0_RFE,
2312 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2313 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2314 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
2315 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
2316 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
2317 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
2320 static void rec_SPECIAL(struct lightrec_cstate *state,
2321 const struct block *block, u16 offset)
2323 union code c = block->opcode_list[offset].c;
2324 lightrec_rec_func_t f = rec_special[c.r.op];
2326 if (!HAS_DEFAULT_ELM && unlikely(!f))
2327 unknown_opcode(state, block, offset);
2329 (*f)(state, block, offset);
2332 static void rec_REGIMM(struct lightrec_cstate *state,
2333 const struct block *block, u16 offset)
2335 union code c = block->opcode_list[offset].c;
2336 lightrec_rec_func_t f = rec_regimm[c.r.rt];
2338 if (!HAS_DEFAULT_ELM && unlikely(!f))
2339 unknown_opcode(state, block, offset);
2341 (*f)(state, block, offset);
2344 static void rec_CP0(struct lightrec_cstate *state,
2345 const struct block *block, u16 offset)
2347 union code c = block->opcode_list[offset].c;
2348 lightrec_rec_func_t f = rec_cp0[c.r.rs];
2350 if (!HAS_DEFAULT_ELM && unlikely(!f))
2351 rec_CP(state, block, offset);
2353 (*f)(state, block, offset);
2356 static void rec_CP2(struct lightrec_cstate *state,
2357 const struct block *block, u16 offset)
2359 union code c = block->opcode_list[offset].c;
2361 if (c.r.op == OP_CP2_BASIC) {
2362 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2364 if (HAS_DEFAULT_ELM || likely(f)) {
2365 (*f)(state, block, offset);
2370 rec_CP(state, block, offset);
2373 void lightrec_rec_opcode(struct lightrec_cstate *state,
2374 const struct block *block, u16 offset)
2376 struct regcache *reg_cache = state->reg_cache;
2377 struct lightrec_branch_target *target;
2378 const struct opcode *op = &block->opcode_list[offset];
2379 jit_state_t *_jit = block->_jit;
2380 lightrec_rec_func_t f;
2382 if (op->flags & LIGHTREC_SYNC) {
2383 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2386 lightrec_storeback_regs(reg_cache, _jit);
2387 lightrec_regcache_reset(reg_cache);
2389 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2390 target = &state->targets[state->nb_targets++];
2391 target->offset = offset;
2392 target->label = jit_indirect();
2395 if (likely(op->opcode)) {
2396 f = rec_standard[op->i.op];
2398 if (!HAS_DEFAULT_ELM && unlikely(!f))
2399 unknown_opcode(state, block, offset);
2401 (*f)(state, block, offset);
2404 if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) {
2405 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true);
2406 pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd));
2408 if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) {
2409 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true);
2410 pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt));
2412 if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) {
2413 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true);
2414 pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt));