lightrec: Increase size of hw registers area
[pcsx_rearmed.git] / deps / lightrec / emitter.c
1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 /*
3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
4  */
5
6 #include "blockcache.h"
7 #include "debug.h"
8 #include "disassembler.h"
9 #include "emitter.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
12 #include "regcache.h"
13
14 #include <stdbool.h>
15 #include <stddef.h>
16
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
18
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
24 static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset);
25 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
26                             const struct block *block, u16 offset, u8 reg, u8 in_reg);
27 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
28                             const struct block *block, u16 offset,
29                             u8 reg, u8 out_reg);
30
31 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
32 {
33         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
34                 block->opcode_list[offset].c.opcode,
35                 block->pc + (offset << 2));
36 }
37
38 static void
39 lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void))
40 {
41         /* Prevent jit_jmpi() from using our cycles register as a temporary */
42         jit_live(LIGHTREC_REG_CYCLE);
43
44         jit_patch_abs(jit_jmpi(), fn);
45 }
46
47 static void
48 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
49 {
50         lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func);
51 }
52
53 static void
54 lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit)
55 {
56         lightrec_jump_to_fn(_jit, state->state->ds_check_func);
57 }
58
59 static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
60                                u8 ra_reg, u32 pc, u32 link)
61 {
62         u8 link_reg;
63
64         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
65         lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
66         lightrec_free_reg(reg_cache, link_reg);
67 }
68
69 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
70                                        const struct block *block, u16 offset,
71                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
72                                        u32 link, bool update_cycles)
73 {
74         struct regcache *reg_cache = state->reg_cache;
75         jit_state_t *_jit = block->_jit;
76         const struct opcode *op = &block->opcode_list[offset],
77                             *ds = get_delay_slot(block->opcode_list, offset);
78         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
79
80         jit_note(__FILE__, __LINE__);
81
82         if (link && ra_reg != reg_new_pc)
83                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
84
85         if (reg_new_pc < 0)
86                 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
87         else
88                 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
89
90         if (link && ra_reg == reg_new_pc) {
91                 /* Handle the special case: JALR $r0, $r0
92                  * In that case the target PC should be the old value of the
93                  * register. */
94                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
95         }
96
97         if (has_delay_slot(op->c) &&
98             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
99                 cycles += lightrec_cycles_of_opcode(ds->c);
100
101                 /* Recompile the delay slot */
102                 if (ds->c.opcode)
103                         lightrec_rec_opcode(state, block, offset + 1);
104         }
105
106         /* Clean the remaining registers */
107         lightrec_clean_regs(reg_cache, _jit);
108
109         if (cycles && update_cycles) {
110                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
111                 pr_debug("EOB: %u cycles\n", cycles);
112         }
113
114         if (op_flag_load_delay(ds->flags)
115             && opcode_is_load(ds->c) && !state->no_load_delay) {
116                 /* If the delay slot is a load opcode, its target register
117                  * will be written after the first opcode of the target is
118                  * executed. Handle this by jumping to a special section of
119                  * the dispatcher. It expects the loaded value to be in
120                  * REG_TEMP, and the target register number to be in JIT_V1.*/
121                 jit_movi(JIT_V1, ds->c.i.rt);
122
123                 lightrec_jump_to_ds_check(state, _jit);
124         } else {
125                 lightrec_jump_to_eob(state, _jit);
126         }
127
128         lightrec_regcache_reset(reg_cache);
129 }
130
131 void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state,
132                                        const struct block *block, u16 offset)
133 {
134         struct regcache *reg_cache = state->reg_cache;
135         jit_state_t *_jit = block->_jit;
136
137         lightrec_clean_regs(reg_cache, _jit);
138
139         /* Call the interpreter with the block's address in JIT_V1 and the
140          * PC (which might have an offset) in JIT_V0. */
141         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
142                           block->pc + (offset << 2));
143         if (lightrec_store_next_pc()) {
144               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
145                          LIGHTREC_REG_STATE, JIT_V0);
146         }
147
148         jit_movi(JIT_V1, (uintptr_t)block);
149
150         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
151         lightrec_jump_to_fn(_jit, state->state->interpreter_func);
152 }
153
154 static void lightrec_emit_eob(struct lightrec_cstate *state,
155                               const struct block *block, u16 offset)
156 {
157         struct regcache *reg_cache = state->reg_cache;
158         jit_state_t *_jit = block->_jit;
159
160         lightrec_clean_regs(reg_cache, _jit);
161
162         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
163                           block->pc + (offset << 2));
164         if (lightrec_store_next_pc()) {
165               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
166                          LIGHTREC_REG_STATE, JIT_V0);
167         }
168
169         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
170
171         lightrec_jump_to_eob(state, _jit);
172 }
173
174 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
175 {
176         union code c = block->opcode_list[offset].c;
177
178         _jit_name(block->_jit, __func__);
179         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
180 }
181
182 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
183 {
184         union code c = block->opcode_list[offset].c;
185
186         _jit_name(block->_jit, __func__);
187         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
188                                    get_branch_pc(block, offset, 2), true);
189 }
190
191 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
192 {
193         union code c = block->opcode_list[offset].c;
194
195         _jit_name(block->_jit, __func__);
196         lightrec_emit_end_of_block(state, block, offset, -1,
197                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
198                                    31, 0, true);
199 }
200
201 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
202 {
203         union code c = block->opcode_list[offset].c;
204
205         _jit_name(block->_jit, __func__);
206         lightrec_emit_end_of_block(state, block, offset, -1,
207                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
208                                    31, get_branch_pc(block, offset, 2), true);
209 }
210
211 static void lightrec_do_early_unload(struct lightrec_cstate *state,
212                                      const struct block *block, u16 offset)
213 {
214         struct regcache *reg_cache = state->reg_cache;
215         const struct opcode *op = &block->opcode_list[offset];
216         jit_state_t *_jit = block->_jit;
217         unsigned int i;
218         u8 reg;
219         struct {
220                 u8 reg, op;
221         } reg_ops[3] = {
222                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
223                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
224                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
225         };
226
227         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
228                 reg = reg_ops[i].reg;
229
230                 switch (reg_ops[i].op) {
231                 case LIGHTREC_REG_UNLOAD:
232                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
233                         break;
234
235                 case LIGHTREC_REG_DISCARD:
236                         lightrec_discard_reg_if_loaded(reg_cache, reg);
237                         break;
238
239                 case LIGHTREC_REG_CLEAN:
240                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
241                         break;
242                 default:
243                         break;
244                 };
245         }
246 }
247
248 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
249                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
250 {
251         struct regcache *reg_cache = state->reg_cache;
252         struct native_register *regs_backup;
253         jit_state_t *_jit = block->_jit;
254         struct lightrec_branch *branch;
255         const struct opcode *op = &block->opcode_list[offset],
256                             *ds = get_delay_slot(block->opcode_list, offset);
257         jit_node_t *addr;
258         bool is_forward = (s16)op->i.imm >= 0;
259         int op_cycles = lightrec_cycles_of_opcode(op->c);
260         u32 target_offset, cycles = state->cycles + op_cycles;
261         bool no_indirection = false;
262         u32 next_pc;
263         u8 rs, rt;
264
265         jit_note(__FILE__, __LINE__);
266
267         if (!op_flag_no_ds(op->flags))
268                 cycles += lightrec_cycles_of_opcode(ds->c);
269
270         state->cycles = -op_cycles;
271
272         if (!unconditional) {
273                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
274                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
275                                                     _jit, op->i.rt, REG_EXT);
276
277                 /* Unload dead registers before evaluating the branch */
278                 if (OPT_EARLY_UNLOAD)
279                         lightrec_do_early_unload(state, block, offset);
280
281                 if (op_flag_local_branch(op->flags) &&
282                     (op_flag_no_ds(op->flags) || !ds->opcode) &&
283                     is_forward && !lightrec_has_dirty_regs(reg_cache))
284                         no_indirection = true;
285
286                 if (no_indirection)
287                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
288         }
289
290         if (cycles)
291                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
292
293         if (!unconditional) {
294                 /* Generate the branch opcode */
295                 if (!no_indirection)
296                         addr = jit_new_node_pww(code, NULL, rs, rt);
297
298                 lightrec_free_regs(reg_cache);
299                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
300         }
301
302         if (op_flag_local_branch(op->flags)) {
303                 /* Recompile the delay slot */
304                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
305                         /* Never handle load delays with local branches. */
306                         state->no_load_delay = true;
307                         lightrec_rec_opcode(state, block, offset + 1);
308                 }
309
310                 if (link)
311                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
312
313                 /* Clean remaining registers */
314                 lightrec_clean_regs(reg_cache, _jit);
315
316                 target_offset = offset + 1 + (s16)op->i.imm
317                         - !!op_flag_no_ds(op->flags);
318                 pr_debug("Adding local branch to offset 0x%x\n",
319                          target_offset << 2);
320                 branch = &state->local_branches[
321                         state->nb_local_branches++];
322
323                 branch->target = target_offset;
324
325                 if (no_indirection)
326                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
327                 else if (is_forward)
328                         branch->branch = jit_b();
329                 else
330                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
331         }
332
333         if (!op_flag_local_branch(op->flags) || !is_forward) {
334                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
335                 state->no_load_delay = op_flag_local_branch(op->flags);
336                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
337                                            31, link, false);
338         }
339
340         if (!unconditional) {
341                 if (!no_indirection)
342                         jit_patch(addr);
343
344                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
345
346                 if (bz && link)
347                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
348
349                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
350                         state->no_load_delay = true;
351                         lightrec_rec_opcode(state, block, offset + 1);
352                 }
353         }
354 }
355
356 static void rec_BNE(struct lightrec_cstate *state,
357                     const struct block *block, u16 offset)
358 {
359         union code c = block->opcode_list[offset].c;
360
361         _jit_name(block->_jit, __func__);
362
363         if (c.i.rt == 0)
364                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
365         else
366                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
367 }
368
369 static void rec_BEQ(struct lightrec_cstate *state,
370                     const struct block *block, u16 offset)
371 {
372         union code c = block->opcode_list[offset].c;
373
374         _jit_name(block->_jit, __func__);
375
376         if (c.i.rt == 0)
377                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
378         else
379                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
380 }
381
382 static void rec_BLEZ(struct lightrec_cstate *state,
383                      const struct block *block, u16 offset)
384 {
385         union code c = block->opcode_list[offset].c;
386
387         _jit_name(block->_jit, __func__);
388         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
389 }
390
391 static void rec_BGTZ(struct lightrec_cstate *state,
392                      const struct block *block, u16 offset)
393 {
394         _jit_name(block->_jit, __func__);
395         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
396 }
397
398 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
399                             const struct block *block, u16 offset)
400 {
401         _jit_name(block->_jit, __func__);
402         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
403 }
404
405 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
406                               const struct block *block, u16 offset)
407 {
408         _jit_name(block->_jit, __func__);
409         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
410               get_branch_pc(block, offset, 2), false, true);
411 }
412
413 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
414                             const struct block *block, u16 offset)
415 {
416         union code c = block->opcode_list[offset].c;
417
418         _jit_name(block->_jit, __func__);
419         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
420 }
421
422 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
423                               const struct block *block, u16 offset)
424 {
425         const struct opcode *op = &block->opcode_list[offset];
426         _jit_name(block->_jit, __func__);
427         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
428               get_branch_pc(block, offset, 2),
429               !op->i.rs, true);
430 }
431
432 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
433                         u16 offset, jit_code_t code, bool slti)
434 {
435         struct regcache *reg_cache = state->reg_cache;
436         union code c = block->opcode_list[offset].c;
437         jit_state_t *_jit = block->_jit;
438         u8 rs, rt, out_flags = REG_EXT;
439
440         if (slti)
441                 out_flags |= REG_ZEXT;
442
443         jit_note(__FILE__, __LINE__);
444         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
445         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
446
447         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
448
449         lightrec_free_reg(reg_cache, rs);
450         lightrec_free_reg(reg_cache, rt);
451 }
452
453 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
454                             u16 offset, jit_code_t code, bool out_ext)
455 {
456         struct regcache *reg_cache = state->reg_cache;
457         union code c = block->opcode_list[offset].c;
458         jit_state_t *_jit = block->_jit;
459         u8 rd, rt, rs;
460
461         jit_note(__FILE__, __LINE__);
462         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
463         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
464         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
465                                     out_ext ? REG_EXT | REG_ZEXT : 0);
466
467         jit_new_node_www(code, rd, rs, rt);
468
469         lightrec_free_reg(reg_cache, rs);
470         lightrec_free_reg(reg_cache, rt);
471         lightrec_free_reg(reg_cache, rd);
472 }
473
474 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
475                            u16 offset, jit_code_t code)
476 {
477         struct regcache *reg_cache = state->reg_cache;
478         union code c = block->opcode_list[offset].c;
479         jit_state_t *_jit = block->_jit;
480         u8 rd, rt, rs, temp, flags = 0;
481
482         jit_note(__FILE__, __LINE__);
483         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
484
485         if (code == jit_code_rshr)
486                 flags = REG_EXT;
487         else if (code == jit_code_rshr_u)
488                 flags = REG_ZEXT;
489
490         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
491         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
492
493         if (rs != rd && rt != rd) {
494                 jit_andi(rd, rs, 0x1f);
495                 jit_new_node_www(code, rd, rt, rd);
496         } else {
497                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
498                 jit_andi(temp, rs, 0x1f);
499                 jit_new_node_www(code, rd, rt, temp);
500                 lightrec_free_reg(reg_cache, temp);
501         }
502
503         lightrec_free_reg(reg_cache, rs);
504         lightrec_free_reg(reg_cache, rt);
505         lightrec_free_reg(reg_cache, rd);
506 }
507
508 static void rec_movi(struct lightrec_cstate *state,
509                      const struct block *block, u16 offset)
510 {
511         struct regcache *reg_cache = state->reg_cache;
512         union code c = block->opcode_list[offset].c;
513         jit_state_t *_jit = block->_jit;
514         u16 flags = REG_EXT;
515         u8 rt;
516
517         if (!(c.i.imm & 0x8000))
518                 flags |= REG_ZEXT;
519
520         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
521
522         jit_movi(rt, (s32)(s16) c.i.imm);
523
524         lightrec_free_reg(reg_cache, rt);
525 }
526
527 static void rec_ADDIU(struct lightrec_cstate *state,
528                       const struct block *block, u16 offset)
529 {
530         _jit_name(block->_jit, __func__);
531
532         if (block->opcode_list[offset].c.i.rs)
533                 rec_alu_imm(state, block, offset, jit_code_addi, false);
534         else
535                 rec_movi(state, block, offset);
536 }
537
538 static void rec_ADDI(struct lightrec_cstate *state,
539                      const struct block *block, u16 offset)
540 {
541         /* TODO: Handle the exception? */
542         _jit_name(block->_jit, __func__);
543         rec_ADDIU(state, block, offset);
544 }
545
546 static void rec_SLTIU(struct lightrec_cstate *state,
547                       const struct block *block, u16 offset)
548 {
549         _jit_name(block->_jit, __func__);
550         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
551 }
552
553 static void rec_SLTI(struct lightrec_cstate *state,
554                      const struct block *block, u16 offset)
555 {
556         _jit_name(block->_jit, __func__);
557         rec_alu_imm(state, block, offset, jit_code_lti, true);
558 }
559
560 static void rec_ANDI(struct lightrec_cstate *state,
561                      const struct block *block, u16 offset)
562 {
563         struct regcache *reg_cache = state->reg_cache;
564         union code c = block->opcode_list[offset].c;
565         jit_state_t *_jit = block->_jit;
566         u8 rs, rt;
567
568         _jit_name(block->_jit, __func__);
569         jit_note(__FILE__, __LINE__);
570         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
571         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
572                                     REG_EXT | REG_ZEXT);
573
574         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
575          * casts to uint8_t / uint16_t. */
576         if (c.i.imm == 0xff)
577                 jit_extr_uc(rt, rs);
578         else if (c.i.imm == 0xffff)
579                 jit_extr_us(rt, rs);
580         else
581                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
582
583         lightrec_free_reg(reg_cache, rs);
584         lightrec_free_reg(reg_cache, rt);
585 }
586
587 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
588                            u16 offset, jit_code_t code)
589 {
590         struct regcache *reg_cache = state->reg_cache;
591         union code c = block->opcode_list[offset].c;
592         jit_state_t *_jit = block->_jit;
593         u8 rs, rt, flags;
594
595         jit_note(__FILE__, __LINE__);
596         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
597         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
598
599         flags = lightrec_get_reg_in_flags(reg_cache, rs);
600         lightrec_set_reg_out_flags(reg_cache, rt, flags);
601
602         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
603
604         lightrec_free_reg(reg_cache, rs);
605         lightrec_free_reg(reg_cache, rt);
606 }
607
608
609 static void rec_ORI(struct lightrec_cstate *state,
610                     const struct block *block, u16 offset)
611 {
612         _jit_name(block->_jit, __func__);
613         rec_alu_or_xor(state, block, offset, jit_code_ori);
614 }
615
616 static void rec_XORI(struct lightrec_cstate *state,
617                      const struct block *block, u16 offset)
618 {
619         _jit_name(block->_jit, __func__);
620         rec_alu_or_xor(state, block, offset, jit_code_xori);
621 }
622
623 static void rec_LUI(struct lightrec_cstate *state,
624                     const struct block *block, u16 offset)
625 {
626         struct regcache *reg_cache = state->reg_cache;
627         union code c = block->opcode_list[offset].c;
628         jit_state_t *_jit = block->_jit;
629         u8 rt, flags = REG_EXT;
630
631         jit_name(__func__);
632         jit_note(__FILE__, __LINE__);
633
634         if (!(c.i.imm & BIT(15)))
635                 flags |= REG_ZEXT;
636
637         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
638
639         jit_movi(rt, (s32)(c.i.imm << 16));
640
641         lightrec_free_reg(reg_cache, rt);
642 }
643
644 static void rec_special_ADDU(struct lightrec_cstate *state,
645                              const struct block *block, u16 offset)
646 {
647         _jit_name(block->_jit, __func__);
648         rec_alu_special(state, block, offset, jit_code_addr, false);
649 }
650
651 static void rec_special_ADD(struct lightrec_cstate *state,
652                             const struct block *block, u16 offset)
653 {
654         /* TODO: Handle the exception? */
655         _jit_name(block->_jit, __func__);
656         rec_alu_special(state, block, offset, jit_code_addr, false);
657 }
658
659 static void rec_special_SUBU(struct lightrec_cstate *state,
660                              const struct block *block, u16 offset)
661 {
662         _jit_name(block->_jit, __func__);
663         rec_alu_special(state, block, offset, jit_code_subr, false);
664 }
665
666 static void rec_special_SUB(struct lightrec_cstate *state,
667                             const struct block *block, u16 offset)
668 {
669         /* TODO: Handle the exception? */
670         _jit_name(block->_jit, __func__);
671         rec_alu_special(state, block, offset, jit_code_subr, false);
672 }
673
674 static void rec_special_AND(struct lightrec_cstate *state,
675                             const struct block *block, u16 offset)
676 {
677         struct regcache *reg_cache = state->reg_cache;
678         union code c = block->opcode_list[offset].c;
679         jit_state_t *_jit = block->_jit;
680         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
681
682         _jit_name(block->_jit, __func__);
683         jit_note(__FILE__, __LINE__);
684         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
685         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
686         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
687
688         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
689         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
690
691         /* Z(rd) = Z(rs) | Z(rt) */
692         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
693
694         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
695         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
696             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
697             (REG_EXT & flags_rs & flags_rt))
698                 flags_rd |= REG_EXT;
699
700         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
701
702         jit_andr(rd, rs, rt);
703
704         lightrec_free_reg(reg_cache, rs);
705         lightrec_free_reg(reg_cache, rt);
706         lightrec_free_reg(reg_cache, rd);
707 }
708
709 static void rec_special_or_nor(struct lightrec_cstate *state,
710                                const struct block *block, u16 offset, bool nor)
711 {
712         struct regcache *reg_cache = state->reg_cache;
713         union code c = block->opcode_list[offset].c;
714         jit_state_t *_jit = block->_jit;
715         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
716
717         jit_note(__FILE__, __LINE__);
718         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
719         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
720         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
721
722         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
723         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
724
725         /* or: Z(rd) = Z(rs) & Z(rt)
726          * nor: Z(rd) = 0 */
727         if (!nor)
728                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
729
730         /* E(rd) = E(rs) & E(rt) */
731         if (REG_EXT & flags_rs & flags_rt)
732                 flags_rd |= REG_EXT;
733
734         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
735
736         jit_orr(rd, rs, rt);
737
738         if (nor)
739                 jit_comr(rd, rd);
740
741         lightrec_free_reg(reg_cache, rs);
742         lightrec_free_reg(reg_cache, rt);
743         lightrec_free_reg(reg_cache, rd);
744 }
745
746 static void rec_special_OR(struct lightrec_cstate *state,
747                            const struct block *block, u16 offset)
748 {
749         _jit_name(block->_jit, __func__);
750         rec_special_or_nor(state, block, offset, false);
751 }
752
753 static void rec_special_NOR(struct lightrec_cstate *state,
754                             const struct block *block, u16 offset)
755 {
756         _jit_name(block->_jit, __func__);
757         rec_special_or_nor(state, block, offset, true);
758 }
759
760 static void rec_special_XOR(struct lightrec_cstate *state,
761                             const struct block *block, u16 offset)
762 {
763         struct regcache *reg_cache = state->reg_cache;
764         union code c = block->opcode_list[offset].c;
765         jit_state_t *_jit = block->_jit;
766         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
767
768         _jit_name(block->_jit, __func__);
769
770         jit_note(__FILE__, __LINE__);
771         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
772         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
773         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
774
775         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
776         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
777
778         /* Z(rd) = Z(rs) & Z(rt) */
779         flags_rd = REG_ZEXT & flags_rs & flags_rt;
780
781         /* E(rd) = E(rs) & E(rt) */
782         flags_rd |= REG_EXT & flags_rs & flags_rt;
783
784         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
785
786         jit_xorr(rd, rs, rt);
787
788         lightrec_free_reg(reg_cache, rs);
789         lightrec_free_reg(reg_cache, rt);
790         lightrec_free_reg(reg_cache, rd);
791 }
792
793 static void rec_special_SLTU(struct lightrec_cstate *state,
794                              const struct block *block, u16 offset)
795 {
796         _jit_name(block->_jit, __func__);
797         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
798 }
799
800 static void rec_special_SLT(struct lightrec_cstate *state,
801                             const struct block *block, u16 offset)
802 {
803         _jit_name(block->_jit, __func__);
804         rec_alu_special(state, block, offset, jit_code_ltr, true);
805 }
806
807 static void rec_special_SLLV(struct lightrec_cstate *state,
808                              const struct block *block, u16 offset)
809 {
810         _jit_name(block->_jit, __func__);
811         rec_alu_shiftv(state, block, offset, jit_code_lshr);
812 }
813
814 static void rec_special_SRLV(struct lightrec_cstate *state,
815                              const struct block *block, u16 offset)
816 {
817         _jit_name(block->_jit, __func__);
818         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
819 }
820
821 static void rec_special_SRAV(struct lightrec_cstate *state,
822                              const struct block *block, u16 offset)
823 {
824         _jit_name(block->_jit, __func__);
825         rec_alu_shiftv(state, block, offset, jit_code_rshr);
826 }
827
828 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
829                           u16 offset, jit_code_t code)
830 {
831         struct regcache *reg_cache = state->reg_cache;
832         union code c = block->opcode_list[offset].c;
833         jit_state_t *_jit = block->_jit;
834         u8 rd, rt, flags = 0;
835
836         jit_note(__FILE__, __LINE__);
837
838         if (code == jit_code_rshi)
839                 flags = REG_EXT;
840         else if (code == jit_code_rshi_u)
841                 flags = REG_ZEXT;
842
843         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
844
845         /* Input reg is zero-extended, if we SRL at least by one bit, we know
846          * the output reg will be both zero-extended and sign-extended. */
847         if (code == jit_code_rshi_u && c.r.imm)
848                 flags |= REG_EXT;
849         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
850
851         jit_new_node_www(code, rd, rt, c.r.imm);
852
853         lightrec_free_reg(reg_cache, rt);
854         lightrec_free_reg(reg_cache, rd);
855 }
856
857 static void rec_special_SLL(struct lightrec_cstate *state,
858                             const struct block *block, u16 offset)
859 {
860         _jit_name(block->_jit, __func__);
861         rec_alu_shift(state, block, offset, jit_code_lshi);
862 }
863
864 static void rec_special_SRL(struct lightrec_cstate *state,
865                             const struct block *block, u16 offset)
866 {
867         _jit_name(block->_jit, __func__);
868         rec_alu_shift(state, block, offset, jit_code_rshi_u);
869 }
870
871 static void rec_special_SRA(struct lightrec_cstate *state,
872                             const struct block *block, u16 offset)
873 {
874         _jit_name(block->_jit, __func__);
875         rec_alu_shift(state, block, offset, jit_code_rshi);
876 }
877
878 static void rec_alu_mult(struct lightrec_cstate *state,
879                          const struct block *block, u16 offset, bool is_signed)
880 {
881         struct regcache *reg_cache = state->reg_cache;
882         union code c = block->opcode_list[offset].c;
883         u32 flags = block->opcode_list[offset].flags;
884         u8 reg_lo = get_mult_div_lo(c);
885         u8 reg_hi = get_mult_div_hi(c);
886         jit_state_t *_jit = block->_jit;
887         u8 lo, hi, rs, rt, rflags = 0;
888
889         jit_note(__FILE__, __LINE__);
890
891         if (is_signed)
892                 rflags = REG_EXT;
893         else
894                 rflags = REG_ZEXT;
895
896         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
897         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
898
899         if (!op_flag_no_lo(flags))
900                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
901         else if (__WORDSIZE == 32)
902                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
903
904         if (!op_flag_no_hi(flags))
905                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
906
907         if (__WORDSIZE == 32) {
908                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
909                  * operation if the MULT was detected a 32-bit only. */
910                 if (!op_flag_no_hi(flags)) {
911                         if (is_signed)
912                                 jit_qmulr(lo, hi, rs, rt);
913                         else
914                                 jit_qmulr_u(lo, hi, rs, rt);
915                 } else {
916                         jit_mulr(lo, rs, rt);
917                 }
918         } else {
919                 /* On 64-bit systems, do a 64*64->64 bit operation. */
920                 if (op_flag_no_lo(flags)) {
921                         jit_mulr(hi, rs, rt);
922                         jit_rshi(hi, hi, 32);
923                 } else {
924                         jit_mulr(lo, rs, rt);
925
926                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
927                         if (!op_flag_no_hi(flags))
928                                 jit_rshi(hi, lo, 32);
929                 }
930         }
931
932         lightrec_free_reg(reg_cache, rs);
933         lightrec_free_reg(reg_cache, rt);
934         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
935                 lightrec_free_reg(reg_cache, lo);
936         if (!op_flag_no_hi(flags))
937                 lightrec_free_reg(reg_cache, hi);
938 }
939
940 static void rec_alu_div(struct lightrec_cstate *state,
941                         const struct block *block, u16 offset, bool is_signed)
942 {
943         struct regcache *reg_cache = state->reg_cache;
944         union code c = block->opcode_list[offset].c;
945         u32 flags = block->opcode_list[offset].flags;
946         bool no_check = op_flag_no_div_check(flags);
947         u8 reg_lo = get_mult_div_lo(c);
948         u8 reg_hi = get_mult_div_hi(c);
949         jit_state_t *_jit = block->_jit;
950         jit_node_t *branch, *to_end;
951         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
952
953         jit_note(__FILE__, __LINE__);
954
955         if (is_signed)
956                 rflags = REG_EXT;
957         else
958                 rflags = REG_ZEXT;
959
960         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
961         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
962
963         if (!op_flag_no_lo(flags))
964                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
965
966         if (!op_flag_no_hi(flags))
967                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
968
969         /* Jump to special handler if dividing by zero  */
970         if (!no_check)
971                 branch = jit_beqi(rt, 0);
972
973         if (op_flag_no_lo(flags)) {
974                 if (is_signed)
975                         jit_remr(hi, rs, rt);
976                 else
977                         jit_remr_u(hi, rs, rt);
978         } else if (op_flag_no_hi(flags)) {
979                 if (is_signed)
980                         jit_divr(lo, rs, rt);
981                 else
982                         jit_divr_u(lo, rs, rt);
983         } else {
984                 if (is_signed)
985                         jit_qdivr(lo, hi, rs, rt);
986                 else
987                         jit_qdivr_u(lo, hi, rs, rt);
988         }
989
990         if (!no_check) {
991                 /* Jump above the div-by-zero handler */
992                 to_end = jit_b();
993
994                 jit_patch(branch);
995
996                 if (!op_flag_no_lo(flags)) {
997                         if (is_signed) {
998                                 jit_ltr(lo, rs, rt);
999                                 jit_lshi(lo, lo, 1);
1000                                 jit_subi(lo, lo, 1);
1001                         } else {
1002                                 jit_subi(lo, rt, 1);
1003                         }
1004                 }
1005
1006                 if (!op_flag_no_hi(flags))
1007                         jit_movr(hi, rs);
1008
1009                 jit_patch(to_end);
1010         }
1011
1012         lightrec_free_reg(reg_cache, rs);
1013         lightrec_free_reg(reg_cache, rt);
1014
1015         if (!op_flag_no_lo(flags))
1016                 lightrec_free_reg(reg_cache, lo);
1017
1018         if (!op_flag_no_hi(flags))
1019                 lightrec_free_reg(reg_cache, hi);
1020 }
1021
1022 static void rec_special_MULT(struct lightrec_cstate *state,
1023                              const struct block *block, u16 offset)
1024 {
1025         _jit_name(block->_jit, __func__);
1026         rec_alu_mult(state, block, offset, true);
1027 }
1028
1029 static void rec_special_MULTU(struct lightrec_cstate *state,
1030                               const struct block *block, u16 offset)
1031 {
1032         _jit_name(block->_jit, __func__);
1033         rec_alu_mult(state, block, offset, false);
1034 }
1035
1036 static void rec_special_DIV(struct lightrec_cstate *state,
1037                             const struct block *block, u16 offset)
1038 {
1039         _jit_name(block->_jit, __func__);
1040         rec_alu_div(state, block, offset, true);
1041 }
1042
1043 static void rec_special_DIVU(struct lightrec_cstate *state,
1044                              const struct block *block, u16 offset)
1045 {
1046         _jit_name(block->_jit, __func__);
1047         rec_alu_div(state, block, offset, false);
1048 }
1049
1050 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1051                              const struct block *block, u8 dst, u8 src)
1052 {
1053         struct regcache *reg_cache = state->reg_cache;
1054         jit_state_t *_jit = block->_jit;
1055
1056         jit_note(__FILE__, __LINE__);
1057         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1058         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1059
1060         jit_extr_i(dst, src);
1061
1062         lightrec_free_reg(reg_cache, src);
1063         lightrec_free_reg(reg_cache, dst);
1064 }
1065
1066 static void rec_special_MFHI(struct lightrec_cstate *state,
1067                              const struct block *block, u16 offset)
1068 {
1069         union code c = block->opcode_list[offset].c;
1070
1071         _jit_name(block->_jit, __func__);
1072         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1073 }
1074
1075 static void rec_special_MTHI(struct lightrec_cstate *state,
1076                              const struct block *block, u16 offset)
1077 {
1078         union code c = block->opcode_list[offset].c;
1079
1080         _jit_name(block->_jit, __func__);
1081         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1082 }
1083
1084 static void rec_special_MFLO(struct lightrec_cstate *state,
1085                              const struct block *block, u16 offset)
1086 {
1087         union code c = block->opcode_list[offset].c;
1088
1089         _jit_name(block->_jit, __func__);
1090         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1091 }
1092
1093 static void rec_special_MTLO(struct lightrec_cstate *state,
1094                              const struct block *block, u16 offset)
1095 {
1096         union code c = block->opcode_list[offset].c;
1097
1098         _jit_name(block->_jit, __func__);
1099         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1100 }
1101
1102 static void call_to_c_wrapper(struct lightrec_cstate *state,
1103                               const struct block *block, u32 arg,
1104                               enum c_wrappers wrapper)
1105 {
1106         struct regcache *reg_cache = state->reg_cache;
1107         jit_state_t *_jit = block->_jit;
1108         s8 tmp, tmp2;
1109
1110         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1111         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1112
1113         tmp = lightrec_get_reg_with_value(reg_cache,
1114                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1115         if (tmp < 0) {
1116                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1117                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1118                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1119
1120                 lightrec_temp_set_value(reg_cache, tmp,
1121                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1122         }
1123
1124         lightrec_free_reg(reg_cache, tmp2);
1125
1126 #ifdef __mips__
1127         /* On MIPS, register t9 is always used as the target register for JALR.
1128          * Therefore if it does not contain the target address we must
1129          * invalidate it. */
1130         if (tmp != _T9)
1131                 lightrec_unload_reg(reg_cache, _jit, _T9);
1132 #endif
1133
1134         jit_prepare();
1135         jit_pushargi(arg);
1136
1137         lightrec_regcache_mark_live(reg_cache, _jit);
1138         jit_callr(tmp);
1139
1140         lightrec_free_reg(reg_cache, tmp);
1141         lightrec_regcache_mark_live(reg_cache, _jit);
1142 }
1143
1144 static void rec_io(struct lightrec_cstate *state,
1145                    const struct block *block, u16 offset,
1146                    bool load_rt, bool read_rt)
1147 {
1148         struct regcache *reg_cache = state->reg_cache;
1149         jit_state_t *_jit = block->_jit;
1150         union code c = block->opcode_list[offset].c;
1151         u32 flags = block->opcode_list[offset].flags;
1152         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1153         u32 lut_entry;
1154         u8 zero;
1155
1156         jit_note(__FILE__, __LINE__);
1157
1158         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1159
1160         if (read_rt && likely(c.i.rt))
1161                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1162         else if (load_rt)
1163                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1164
1165         if (op_flag_load_delay(flags) && !state->no_load_delay) {
1166                 /* Clear state->in_delay_slot_n. This notifies the lightrec_rw
1167                  * wrapper that it should write the REG_TEMP register instead of
1168                  * the actual output register of the opcode. */
1169                 zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1170                 jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n),
1171                             LIGHTREC_REG_STATE, zero);
1172                 lightrec_free_reg(reg_cache, zero);
1173         }
1174
1175         if (is_tagged) {
1176                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1177         } else {
1178                 lut_entry = lightrec_get_lut_entry(block);
1179                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1180                                   C_WRAPPER_RW_GENERIC);
1181         }
1182 }
1183
1184 static u32 rec_ram_mask(struct lightrec_state *state)
1185 {
1186         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1187 }
1188
1189 static u32 rec_io_mask(const struct lightrec_state *state)
1190 {
1191         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1192
1193         return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0);
1194 }
1195
1196 static void rec_store_memory(struct lightrec_cstate *cstate,
1197                              const struct block *block,
1198                              u16 offset, jit_code_t code,
1199                              jit_code_t swap_code,
1200                              uintptr_t addr_offset, u32 addr_mask,
1201                              bool invalidate)
1202 {
1203         const struct lightrec_state *state = cstate->state;
1204         struct regcache *reg_cache = cstate->reg_cache;
1205         struct opcode *op = &block->opcode_list[offset];
1206         jit_state_t *_jit = block->_jit;
1207         union code c = op->c;
1208         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1209         s16 imm = (s16)c.i.imm;
1210         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1211         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1212         bool no_mask = op_flag_no_mask(op->flags);
1213         bool add_imm = c.i.imm &&
1214                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1215                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1216         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1217         bool swc2 = c.i.op == OP_SWC2;
1218         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1219
1220         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1221         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1222         if (need_tmp)
1223                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1224
1225         addr_reg = rs;
1226
1227         if (add_imm) {
1228                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1229                 lightrec_free_reg(reg_cache, rs);
1230                 addr_reg = tmp;
1231                 imm = 0;
1232         } else if (simm) {
1233                 lut_offt += simm;
1234         }
1235
1236         if (!no_mask) {
1237                 jit_andi(tmp, addr_reg, addr_mask);
1238                 addr_reg = tmp;
1239         }
1240
1241         if (addr_offset) {
1242                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1243                 jit_addi(tmp2, addr_reg, addr_offset);
1244                 addr_reg2 = tmp2;
1245         } else {
1246                 addr_reg2 = addr_reg;
1247         }
1248
1249         if (is_big_endian() && swap_code && in_reg) {
1250                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1251
1252                 jit_new_node_ww(swap_code, tmp3, rt);
1253                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1254
1255                 lightrec_free_reg(reg_cache, tmp3);
1256         } else {
1257                 jit_new_node_www(code, imm, addr_reg2, rt);
1258         }
1259
1260         lightrec_free_reg(reg_cache, rt);
1261
1262         if (invalidate) {
1263                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1264
1265                 if (c.i.op != OP_SW) {
1266                         jit_andi(tmp, addr_reg, ~3);
1267                         addr_reg = tmp;
1268                 }
1269
1270                 if (!lut_is_32bit(state)) {
1271                         jit_lshi(tmp, addr_reg, 1);
1272                         addr_reg = tmp;
1273                 }
1274
1275                 if (addr_reg == rs && c.i.rs == 0) {
1276                         addr_reg = LIGHTREC_REG_STATE;
1277                 } else {
1278                         jit_add_state(tmp, addr_reg);
1279                         addr_reg = tmp;
1280                 }
1281
1282                 if (lut_is_32bit(state))
1283                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1284                 else
1285                         jit_stxi(lut_offt, addr_reg, tmp3);
1286
1287                 lightrec_free_reg(reg_cache, tmp3);
1288         }
1289
1290         if (addr_offset)
1291                 lightrec_free_reg(reg_cache, tmp2);
1292         if (need_tmp)
1293                 lightrec_free_reg(reg_cache, tmp);
1294         lightrec_free_reg(reg_cache, rs);
1295 }
1296
1297 static void rec_store_ram(struct lightrec_cstate *cstate,
1298                           const struct block *block,
1299                           u16 offset, jit_code_t code,
1300                           jit_code_t swap_code, bool invalidate)
1301 {
1302         struct lightrec_state *state = cstate->state;
1303
1304         _jit_note(block->_jit, __FILE__, __LINE__);
1305
1306         return rec_store_memory(cstate, block, offset, code, swap_code,
1307                                 state->offset_ram, rec_ram_mask(state),
1308                                 invalidate);
1309 }
1310
1311 static void rec_store_scratch(struct lightrec_cstate *cstate,
1312                               const struct block *block, u16 offset,
1313                               jit_code_t code, jit_code_t swap_code)
1314 {
1315         _jit_note(block->_jit, __FILE__, __LINE__);
1316
1317         return rec_store_memory(cstate, block, offset, code, swap_code,
1318                                 cstate->state->offset_scratch,
1319                                 0x1fffffff, false);
1320 }
1321
1322 static void rec_store_io(struct lightrec_cstate *cstate,
1323                          const struct block *block, u16 offset,
1324                          jit_code_t code, jit_code_t swap_code)
1325 {
1326         _jit_note(block->_jit, __FILE__, __LINE__);
1327
1328         return rec_store_memory(cstate, block, offset, code, swap_code,
1329                                 cstate->state->offset_io,
1330                                 rec_io_mask(cstate->state), false);
1331 }
1332
1333 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1334                                            const struct block *block,
1335                                            u16 offset, jit_code_t code,
1336                                            jit_code_t swap_code)
1337 {
1338         struct lightrec_state *state = cstate->state;
1339         struct regcache *reg_cache = cstate->reg_cache;
1340         union code c = block->opcode_list[offset].c;
1341         jit_state_t *_jit = block->_jit;
1342         jit_node_t *to_not_ram, *to_end;
1343         bool swc2 = c.i.op == OP_SWC2;
1344         bool offset_ram_or_scratch = state->offset_ram || state->offset_scratch;
1345         u8 tmp, tmp2, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt;
1346         s16 imm;
1347
1348         jit_note(__FILE__, __LINE__);
1349         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1350         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1351
1352         if (offset_ram_or_scratch)
1353                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1354
1355         /* Convert to KUNSEG and avoid RAM mirrors */
1356         if (state->mirrors_mapped) {
1357                 imm = (s16)c.i.imm;
1358                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1359         } else if (c.i.imm) {
1360                 imm = 0;
1361                 jit_addi(tmp, rs, (s16)c.i.imm);
1362                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1363         } else {
1364                 imm = 0;
1365                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1366         }
1367
1368         lightrec_free_reg(reg_cache, rs);
1369
1370         if (state->offset_ram != state->offset_scratch) {
1371                 to_not_ram = jit_bmsi(tmp, BIT(28));
1372
1373                 jit_movi(tmp2, state->offset_ram);
1374
1375                 to_end = jit_b();
1376                 jit_patch(to_not_ram);
1377
1378                 jit_movi(tmp2, state->offset_scratch);
1379                 jit_patch(to_end);
1380         } else if (state->offset_ram) {
1381                 jit_movi(tmp2, state->offset_ram);
1382         }
1383
1384         if (offset_ram_or_scratch) {
1385                 jit_addr(tmp, tmp, tmp2);
1386                 lightrec_free_reg(reg_cache, tmp2);
1387         }
1388
1389         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1390
1391         if (is_big_endian() && swap_code && in_reg) {
1392                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1393
1394                 jit_new_node_ww(swap_code, tmp2, rt);
1395                 jit_new_node_www(code, imm, tmp, tmp2);
1396
1397                 lightrec_free_reg(reg_cache, tmp2);
1398         } else {
1399                 jit_new_node_www(code, imm, tmp, rt);
1400         }
1401
1402         lightrec_free_reg(reg_cache, rt);
1403         lightrec_free_reg(reg_cache, tmp);
1404 }
1405
1406 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1407                              u16 offset, jit_code_t code, jit_code_t swap_code)
1408 {
1409         struct lightrec_state *state = cstate->state;
1410         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1411         struct regcache *reg_cache = cstate->reg_cache;
1412         union code c = block->opcode_list[offset].c;
1413         jit_state_t *_jit = block->_jit;
1414         jit_node_t *to_not_ram, *to_end;
1415         bool swc2 = c.i.op == OP_SWC2;
1416         u8 tmp, tmp2, tmp3, masked_reg, rs, rt;
1417         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1418
1419         jit_note(__FILE__, __LINE__);
1420
1421         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1422         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1423         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1424
1425         /* Convert to KUNSEG and avoid RAM mirrors */
1426         if (c.i.imm) {
1427                 jit_addi(tmp2, rs, (s16)c.i.imm);
1428                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1429         } else {
1430                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1431         }
1432
1433         lightrec_free_reg(reg_cache, rs);
1434         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1435
1436         if (state->offset_ram != state->offset_scratch) {
1437                 to_not_ram = jit_bgti(tmp2, ram_size);
1438                 masked_reg = tmp2;
1439         } else {
1440                 jit_lti_u(tmp, tmp2, ram_size);
1441                 jit_movnr(tmp, tmp2, tmp);
1442                 masked_reg = tmp;
1443         }
1444
1445         /* Compute the offset to the code LUT */
1446         if (c.i.op == OP_SW)
1447                 jit_andi(tmp, masked_reg, RAM_SIZE - 1);
1448         else
1449                 jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3);
1450
1451         if (!lut_is_32bit(state))
1452                 jit_lshi(tmp, tmp, 1);
1453         jit_add_state(tmp, tmp);
1454
1455         /* Write NULL to the code LUT to invalidate any block that's there */
1456         if (lut_is_32bit(state))
1457                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1458         else
1459                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1460
1461         if (state->offset_ram != state->offset_scratch) {
1462                 jit_movi(tmp, state->offset_ram);
1463
1464                 to_end = jit_b();
1465                 jit_patch(to_not_ram);
1466         }
1467
1468         if (state->offset_ram || state->offset_scratch)
1469                 jit_movi(tmp, state->offset_scratch);
1470
1471         if (state->offset_ram != state->offset_scratch)
1472                 jit_patch(to_end);
1473
1474         if (state->offset_ram || state->offset_scratch)
1475                 jit_addr(tmp2, tmp2, tmp);
1476
1477         lightrec_free_reg(reg_cache, tmp);
1478         lightrec_free_reg(reg_cache, tmp3);
1479
1480         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1481
1482         if (is_big_endian() && swap_code && in_reg) {
1483                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1484
1485                 jit_new_node_ww(swap_code, tmp, rt);
1486                 jit_new_node_www(code, 0, tmp2, tmp);
1487
1488                 lightrec_free_reg(reg_cache, tmp);
1489         } else {
1490                 jit_new_node_www(code, 0, tmp2, rt);
1491         }
1492
1493         lightrec_free_reg(reg_cache, rt);
1494         lightrec_free_reg(reg_cache, tmp2);
1495 }
1496
1497 static void rec_store(struct lightrec_cstate *state,
1498                       const struct block *block, u16 offset,
1499                       jit_code_t code, jit_code_t swap_code)
1500 {
1501         u32 flags = block->opcode_list[offset].flags;
1502         u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1503         bool no_invalidate = op_flag_no_invalidate(flags) ||
1504                 state->state->invalidate_from_dma_only;
1505         union code c = block->opcode_list[offset].c;
1506         bool is_swc2 = c.i.op == OP_SWC2;
1507
1508         if (is_swc2) {
1509                 switch (mode) {
1510                 case LIGHTREC_IO_RAM:
1511                 case LIGHTREC_IO_SCRATCH:
1512                 case LIGHTREC_IO_DIRECT:
1513                 case LIGHTREC_IO_DIRECT_HW:
1514                         rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP);
1515                         break;
1516                 default:
1517                         break;
1518                 }
1519         }
1520
1521         switch (mode) {
1522         case LIGHTREC_IO_RAM:
1523                 rec_store_ram(state, block, offset, code,
1524                               swap_code, !no_invalidate);
1525                 break;
1526         case LIGHTREC_IO_SCRATCH:
1527                 rec_store_scratch(state, block, offset, code, swap_code);
1528                 break;
1529         case LIGHTREC_IO_DIRECT:
1530                 if (no_invalidate) {
1531                         rec_store_direct_no_invalidate(state, block, offset,
1532                                                        code, swap_code);
1533                 } else {
1534                         rec_store_direct(state, block, offset, code, swap_code);
1535                 }
1536                 break;
1537         case LIGHTREC_IO_DIRECT_HW:
1538                 rec_store_io(state, block, offset, code, swap_code);
1539                 break;
1540         default:
1541                 rec_io(state, block, offset, true, false);
1542                 return;
1543         }
1544
1545         if (is_swc2)
1546                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1547 }
1548
1549 static void rec_SB(struct lightrec_cstate *state,
1550                    const struct block *block, u16 offset)
1551 {
1552         _jit_name(block->_jit, __func__);
1553         rec_store(state, block, offset, jit_code_stxi_c, 0);
1554 }
1555
1556 static void rec_SH(struct lightrec_cstate *state,
1557                    const struct block *block, u16 offset)
1558 {
1559         _jit_name(block->_jit, __func__);
1560         rec_store(state, block, offset,
1561                   jit_code_stxi_s, jit_code_bswapr_us);
1562 }
1563
1564 static void rec_SW(struct lightrec_cstate *state,
1565                    const struct block *block, u16 offset)
1566
1567 {
1568         union code c = block->opcode_list[offset].c;
1569
1570         _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
1571         rec_store(state, block, offset,
1572                   jit_code_stxi_i, jit_code_bswapr_ui);
1573 }
1574
1575 static void rec_SWL(struct lightrec_cstate *state,
1576                     const struct block *block, u16 offset)
1577 {
1578         _jit_name(block->_jit, __func__);
1579         rec_io(state, block, offset, true, false);
1580 }
1581
1582 static void rec_SWR(struct lightrec_cstate *state,
1583                     const struct block *block, u16 offset)
1584 {
1585         _jit_name(block->_jit, __func__);
1586         rec_io(state, block, offset, true, false);
1587 }
1588
1589 static void rec_load_memory(struct lightrec_cstate *cstate,
1590                             const struct block *block, u16 offset,
1591                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1592                             uintptr_t addr_offset, u32 addr_mask)
1593 {
1594         struct regcache *reg_cache = cstate->reg_cache;
1595         struct opcode *op = &block->opcode_list[offset];
1596         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1597         jit_state_t *_jit = block->_jit;
1598         u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
1599         bool no_mask = op_flag_no_mask(op->flags);
1600         union code c = op->c;
1601         s16 imm;
1602
1603         if (load_delay || c.i.op == OP_LWC2)
1604                 out_reg = REG_TEMP;
1605         else if (c.i.rt)
1606                 out_reg = c.i.rt;
1607         else
1608                 return;
1609
1610         if (is_unsigned)
1611                 flags |= REG_ZEXT;
1612
1613         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1614         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1615
1616         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1617                 jit_addi(rt, rs, (s16)c.i.imm);
1618                 addr_reg = rt;
1619                 imm = 0;
1620         } else {
1621                 addr_reg = rs;
1622                 imm = (s16)c.i.imm;
1623         }
1624
1625         if (!no_mask) {
1626                 jit_andi(rt, addr_reg, addr_mask);
1627                 addr_reg = rt;
1628         }
1629
1630         if (addr_offset) {
1631                 jit_addi(rt, addr_reg, addr_offset);
1632                 addr_reg = rt;
1633         }
1634
1635         jit_new_node_www(code, rt, addr_reg, imm);
1636
1637         if (is_big_endian() && swap_code) {
1638                 jit_new_node_ww(swap_code, rt, rt);
1639
1640                 if (c.i.op == OP_LH)
1641                         jit_extr_s(rt, rt);
1642                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1643                         jit_extr_i(rt, rt);
1644         }
1645
1646         lightrec_free_reg(reg_cache, rs);
1647         lightrec_free_reg(reg_cache, rt);
1648 }
1649
1650 static void rec_load_ram(struct lightrec_cstate *cstate,
1651                          const struct block *block, u16 offset,
1652                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1653 {
1654         _jit_note(block->_jit, __FILE__, __LINE__);
1655
1656         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1657                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1658 }
1659
1660 static void rec_load_bios(struct lightrec_cstate *cstate,
1661                           const struct block *block, u16 offset,
1662                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1663 {
1664         _jit_note(block->_jit, __FILE__, __LINE__);
1665
1666         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1667                         cstate->state->offset_bios, 0x1fffffff);
1668 }
1669
1670 static void rec_load_scratch(struct lightrec_cstate *cstate,
1671                              const struct block *block, u16 offset,
1672                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1673 {
1674         _jit_note(block->_jit, __FILE__, __LINE__);
1675
1676         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1677                         cstate->state->offset_scratch, 0x1fffffff);
1678 }
1679
1680 static void rec_load_io(struct lightrec_cstate *cstate,
1681                         const struct block *block, u16 offset,
1682                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1683 {
1684         _jit_note(block->_jit, __FILE__, __LINE__);
1685
1686         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1687                         cstate->state->offset_io, rec_io_mask(cstate->state));
1688 }
1689
1690 static void rec_load_direct(struct lightrec_cstate *cstate,
1691                             const struct block *block, u16 offset,
1692                             jit_code_t code, jit_code_t swap_code,
1693                             bool is_unsigned)
1694 {
1695         struct lightrec_state *state = cstate->state;
1696         struct regcache *reg_cache = cstate->reg_cache;
1697         struct opcode *op = &block->opcode_list[offset];
1698         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1699         jit_state_t *_jit = block->_jit;
1700         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1701         u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
1702         union code c = op->c;
1703         s16 imm;
1704
1705         if (load_delay || c.i.op == OP_LWC2)
1706                 out_reg = REG_TEMP;
1707         else if (c.i.rt)
1708                 out_reg = c.i.rt;
1709         else
1710                 return;
1711
1712         if (is_unsigned)
1713                 flags |= REG_ZEXT;
1714
1715         jit_note(__FILE__, __LINE__);
1716         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1717         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1718
1719         if ((state->offset_ram == state->offset_bios &&
1720             state->offset_ram == state->offset_scratch &&
1721             state->mirrors_mapped) || !c.i.imm) {
1722                 addr_reg = rs;
1723                 imm = (s16)c.i.imm;
1724         } else {
1725                 jit_addi(rt, rs, (s16)c.i.imm);
1726                 addr_reg = rt;
1727                 imm = 0;
1728
1729                 if (c.i.rs != c.i.rt)
1730                         lightrec_free_reg(reg_cache, rs);
1731         }
1732
1733         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1734
1735         if (state->offset_ram == state->offset_bios &&
1736             state->offset_ram == state->offset_scratch) {
1737                 if (!state->mirrors_mapped) {
1738                         jit_andi(tmp, addr_reg, BIT(28));
1739                         jit_rshi_u(tmp, tmp, 28 - 22);
1740                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1741                         jit_andr(rt, addr_reg, tmp);
1742                 } else {
1743                         jit_andi(rt, addr_reg, 0x1fffffff);
1744                 }
1745
1746                 if (state->offset_ram)
1747                         jit_movi(tmp, state->offset_ram);
1748         } else {
1749                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1750
1751                 /* Convert to KUNSEG and avoid RAM mirrors */
1752                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1753
1754                 if (state->offset_ram)
1755                         jit_movi(tmp, state->offset_ram);
1756
1757                 to_end = jit_b();
1758
1759                 jit_patch(to_not_ram);
1760
1761                 if (state->offset_bios != state->offset_scratch)
1762                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1763
1764                 /* Convert to KUNSEG */
1765                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1766
1767                 jit_movi(tmp, state->offset_bios);
1768
1769                 if (state->offset_bios != state->offset_scratch) {
1770                         to_end2 = jit_b();
1771
1772                         jit_patch(to_not_bios);
1773
1774                         /* Convert to KUNSEG */
1775                         jit_andi(rt, addr_reg, 0x1f800fff);
1776
1777                         if (state->offset_scratch)
1778                                 jit_movi(tmp, state->offset_scratch);
1779
1780                         jit_patch(to_end2);
1781                 }
1782
1783                 jit_patch(to_end);
1784         }
1785
1786         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1787                 jit_addr(rt, rt, tmp);
1788
1789         jit_new_node_www(code, rt, rt, imm);
1790
1791         if (is_big_endian() && swap_code) {
1792                 jit_new_node_ww(swap_code, rt, rt);
1793
1794                 if (c.i.op == OP_LH)
1795                         jit_extr_s(rt, rt);
1796                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1797                         jit_extr_i(rt, rt);
1798         }
1799
1800         lightrec_free_reg(reg_cache, addr_reg);
1801         lightrec_free_reg(reg_cache, rt);
1802         lightrec_free_reg(reg_cache, tmp);
1803 }
1804
1805 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1806                      u16 offset, jit_code_t code, jit_code_t swap_code,
1807                      bool is_unsigned)
1808 {
1809         const struct opcode *op = &block->opcode_list[offset];
1810         u32 flags = op->flags;
1811
1812         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1813         case LIGHTREC_IO_RAM:
1814                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1815                 break;
1816         case LIGHTREC_IO_BIOS:
1817                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1818                 break;
1819         case LIGHTREC_IO_SCRATCH:
1820                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1821                 break;
1822         case LIGHTREC_IO_DIRECT_HW:
1823                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1824                 break;
1825         case LIGHTREC_IO_DIRECT:
1826                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1827                 break;
1828         default:
1829                 rec_io(state, block, offset, false, true);
1830                 return;
1831         }
1832
1833         if (op->i.op == OP_LWC2) {
1834                 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP);
1835                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1836         }
1837 }
1838
1839 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1840 {
1841         _jit_name(block->_jit, __func__);
1842         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1843 }
1844
1845 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1846 {
1847         _jit_name(block->_jit, __func__);
1848         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1849 }
1850
1851 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1852 {
1853         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1854
1855         _jit_name(block->_jit, __func__);
1856         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1857 }
1858
1859 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1860 {
1861         _jit_name(block->_jit, __func__);
1862         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1863 }
1864
1865 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1866 {
1867         _jit_name(block->_jit, __func__);
1868         rec_io(state, block, offset, true, true);
1869 }
1870
1871 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1872 {
1873         _jit_name(block->_jit, __func__);
1874         rec_io(state, block, offset, true, true);
1875 }
1876
1877 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1878 {
1879         union code c = block->opcode_list[offset].c;
1880         jit_code_t code;
1881
1882         if (is_big_endian() && __WORDSIZE == 64)
1883                 code = jit_code_ldxi_ui;
1884         else
1885                 code = jit_code_ldxi_i;
1886
1887         _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
1888         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1889 }
1890
1891 static void rec_break_syscall(struct lightrec_cstate *state,
1892                               const struct block *block, u16 offset,
1893                               u32 exit_code)
1894 {
1895         struct regcache *reg_cache = state->reg_cache;
1896         jit_state_t *_jit = block->_jit;
1897         u8 tmp;
1898
1899         _jit_note(block->_jit, __FILE__, __LINE__);
1900
1901         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1902
1903         jit_movi(tmp, exit_code);
1904         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1905                    LIGHTREC_REG_STATE, tmp);
1906
1907         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1908                    offsetof(struct lightrec_state, target_cycle));
1909         jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE);
1910         jit_movi(LIGHTREC_REG_CYCLE, 0);
1911         jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
1912                    LIGHTREC_REG_STATE, tmp);
1913         jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
1914                    LIGHTREC_REG_STATE, tmp);
1915
1916         lightrec_free_reg(reg_cache, tmp);
1917
1918         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1919         lightrec_emit_end_of_block(state, block, offset, -1,
1920                                    get_ds_pc(block, offset, 0),
1921                                    31, 0, true);
1922 }
1923
1924 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1925                                 const struct block *block, u16 offset)
1926 {
1927         _jit_name(block->_jit, __func__);
1928         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1929 }
1930
1931 static void rec_special_BREAK(struct lightrec_cstate *state,
1932                               const struct block *block, u16 offset)
1933 {
1934         _jit_name(block->_jit, __func__);
1935         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1936 }
1937
1938 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1939 {
1940         struct regcache *reg_cache = state->reg_cache;
1941         union code c = block->opcode_list[offset].c;
1942         jit_state_t *_jit = block->_jit;
1943
1944         jit_note(__FILE__, __LINE__);
1945
1946         if (c.i.op != OP_SWC2)
1947                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1948
1949         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1950 }
1951
1952 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1953 {
1954         struct regcache *reg_cache = state->reg_cache;
1955         union code c = block->opcode_list[offset].c;
1956         jit_state_t *_jit = block->_jit;
1957
1958         jit_note(__FILE__, __LINE__);
1959         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1960         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1961         lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false);
1962
1963         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1964
1965         if (c.i.op == OP_CP0 &&
1966             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1967             (c.r.rd == 12 || c.r.rd == 13))
1968                 lightrec_emit_end_of_block(state, block, offset, -1,
1969                                            get_ds_pc(block, offset, 1),
1970                                            0, 0, true);
1971 }
1972
1973 static void
1974 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1975 {
1976         struct regcache *reg_cache = state->reg_cache;
1977         union code c = block->opcode_list[offset].c;
1978         jit_state_t *_jit = block->_jit;
1979         u8 rt;
1980
1981         jit_note(__FILE__, __LINE__);
1982
1983         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1984
1985         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1986                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1987
1988         lightrec_free_reg(reg_cache, rt);
1989 }
1990
1991 static bool block_uses_icache(const struct lightrec_cstate *state,
1992                               const struct block *block)
1993 {
1994         const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM];
1995         u32 pc = kunseg(block->pc);
1996
1997         if (pc < map->pc || pc >= map->pc + map->length)
1998                 return false;
1999
2000         return (block->pc >> 28) < 0xa;
2001 }
2002
2003 static void
2004 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
2005 {
2006         struct regcache *reg_cache = state->reg_cache;
2007         const union code c = block->opcode_list[offset].c;
2008         jit_state_t *_jit = block->_jit;
2009         u8 rt, tmp = 0, tmp2, status;
2010         jit_node_t *to_end;
2011
2012         jit_note(__FILE__, __LINE__);
2013
2014         switch(c.r.rd) {
2015         case 1:
2016         case 4:
2017         case 8:
2018         case 14:
2019         case 15:
2020                 /* Those registers are read-only */
2021                 return;
2022         default:
2023                 break;
2024         }
2025
2026         if (!block_uses_icache(state, block) && c.r.rd == 12) {
2027                 /* If we are not running code from the RAM through kuseg or
2028                  * kseg0, handle writes to the Status register in C; as the
2029                  * code may toggle bit 16 which isolates the cache. Code
2030                  * running from kuseg or kseg0 in RAM cannot do that. */
2031                 rec_mtc(state, block, offset);
2032                 return;
2033         }
2034
2035         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
2036
2037         if (c.r.rd != 13) {
2038                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
2039                            LIGHTREC_REG_STATE, rt);
2040         }
2041
2042         if (c.r.rd == 12 || c.r.rd == 13) {
2043                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2044                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2045                            offsetof(struct lightrec_state, regs.cp0[13]));
2046
2047                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2048         }
2049
2050         if (c.r.rd == 12) {
2051                 status = rt;
2052         } else if (c.r.rd == 13) {
2053                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
2054                 jit_andi(tmp2, rt, 0x0300);
2055                 jit_ori(tmp, tmp, 0x0300);
2056                 jit_xori(tmp, tmp, 0x0300);
2057                 jit_orr(tmp, tmp, tmp2);
2058                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2059                            offsetof(struct lightrec_state, regs.cp0[12]));
2060                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
2061                            LIGHTREC_REG_STATE, tmp);
2062                 status = tmp2;
2063         }
2064
2065         if (c.r.rd == 12 || c.r.rd == 13) {
2066                 /* Exit dynarec in case there's a software interrupt.
2067                  * exit_flags = !!(status & tmp & 0x0300) & status; */
2068                 jit_andr(tmp, tmp, status);
2069                 jit_andi(tmp, tmp, 0x0300);
2070                 jit_nei(tmp, tmp, 0);
2071                 jit_andr(tmp, tmp, status);
2072         }
2073
2074         if (c.r.rd == 12) {
2075                 /* Exit dynarec in case we unmask a hardware interrupt.
2076                  * exit_flags = !(~status & 0x401) */
2077
2078                 jit_comr(tmp2, status);
2079                 jit_andi(tmp2, tmp2, 0x401);
2080                 jit_eqi(tmp2, tmp2, 0);
2081                 jit_orr(tmp, tmp, tmp2);
2082         }
2083
2084         lightrec_free_reg(reg_cache, rt);
2085
2086         if (c.r.rd == 12 || c.r.rd == 13) {
2087                 to_end = jit_beqi(tmp, 0);
2088
2089                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2090                            offsetof(struct lightrec_state, target_cycle));
2091                 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2092                 jit_movi(LIGHTREC_REG_CYCLE, 0);
2093                 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2094                            LIGHTREC_REG_STATE, tmp2);
2095                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2096                            LIGHTREC_REG_STATE, tmp2);
2097
2098
2099                 jit_patch(to_end);
2100         }
2101
2102         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
2103             (c.r.rd == 12 || c.r.rd == 13)) {
2104                 state->cycles += lightrec_cycles_of_opcode(c);
2105                 lightrec_emit_eob(state, block, offset + 1);
2106         }
2107 }
2108
2109 static void rec_cp0_MFC0(struct lightrec_cstate *state,
2110                          const struct block *block, u16 offset)
2111 {
2112         _jit_name(block->_jit, __func__);
2113         rec_mfc0(state, block, offset);
2114 }
2115
2116 static void rec_cp0_CFC0(struct lightrec_cstate *state,
2117                          const struct block *block, u16 offset)
2118 {
2119         _jit_name(block->_jit, __func__);
2120         rec_mfc0(state, block, offset);
2121 }
2122
2123 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2124                          const struct block *block, u16 offset)
2125 {
2126         _jit_name(block->_jit, __func__);
2127         rec_mtc0(state, block, offset);
2128 }
2129
2130 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2131                          const struct block *block, u16 offset)
2132 {
2133         _jit_name(block->_jit, __func__);
2134         rec_mtc0(state, block, offset);
2135 }
2136
2137 static unsigned int cp2d_i_offset(u8 reg)
2138 {
2139         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2140 }
2141
2142 static unsigned int cp2d_s_offset(u8 reg)
2143 {
2144         return cp2d_i_offset(reg) + is_big_endian() * 2;
2145 }
2146
2147 static unsigned int cp2c_i_offset(u8 reg)
2148 {
2149         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2150 }
2151
2152 static unsigned int cp2c_s_offset(u8 reg)
2153 {
2154         return cp2c_i_offset(reg) + is_big_endian() * 2;
2155 }
2156
2157 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2158                             const struct block *block, u16 offset,
2159                             u8 reg, u8 out_reg)
2160 {
2161         struct regcache *reg_cache = state->reg_cache;
2162         jit_state_t *_jit = block->_jit;
2163         const u32 zext_regs = 0x300f0080;
2164         u8 rt, tmp, tmp2, tmp3, out, flags;
2165         unsigned int i;
2166
2167         _jit_name(block->_jit, __func__);
2168
2169         if (state->state->ops.cop2_notify) {
2170                 /* We must call cop2_notify, handle that in C. */
2171                 rec_mfc(state, block, offset);
2172                 return;
2173         }
2174
2175         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2176         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2177
2178         if (reg == 15)
2179                 reg = 14;
2180
2181         switch (reg) {
2182         case 1:
2183         case 3:
2184         case 5:
2185         case 8:
2186         case 9:
2187         case 10:
2188         case 11:
2189                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2190                 break;
2191         case 7:
2192         case 16:
2193         case 17:
2194         case 18:
2195         case 19:
2196                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2197                 break;
2198         case 28:
2199         case 29:
2200                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2201                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2202                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2203
2204                 for (i = 0; i < 3; i++) {
2205                         out = i == 0 ? rt : tmp;
2206
2207                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2208                         jit_movi(tmp2, 0x1f);
2209                         jit_rshi(out, tmp, 7);
2210
2211                         jit_ltr(tmp3, tmp2, out);
2212                         jit_movnr(out, tmp2, tmp3);
2213
2214                         jit_gei(tmp2, out, 0);
2215                         jit_movzr(out, tmp2, tmp2);
2216
2217                         if (i > 0) {
2218                                 jit_lshi(tmp, tmp, 5 * i);
2219                                 jit_orr(rt, rt, tmp);
2220                         }
2221                 }
2222
2223
2224                 lightrec_free_reg(reg_cache, tmp);
2225                 lightrec_free_reg(reg_cache, tmp2);
2226                 lightrec_free_reg(reg_cache, tmp3);
2227                 break;
2228         default:
2229                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2230                 break;
2231         }
2232
2233         lightrec_free_reg(reg_cache, rt);
2234 }
2235
2236 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2237                                const struct block *block, u16 offset)
2238 {
2239         const union code c = block->opcode_list[offset].c;
2240
2241         rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2242 }
2243
2244 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2245                                const struct block *block, u16 offset)
2246 {
2247         struct regcache *reg_cache = state->reg_cache;
2248         const union code c = block->opcode_list[offset].c;
2249         jit_state_t *_jit = block->_jit;
2250         u8 rt;
2251
2252         _jit_name(block->_jit, __func__);
2253
2254         if (state->state->ops.cop2_notify) {
2255                 /* We must call cop2_notify, handle that in C. */
2256                 rec_mfc(state, block, offset);
2257                 return;
2258         }
2259
2260         switch (c.r.rd) {
2261         case 4:
2262         case 12:
2263         case 20:
2264         case 26:
2265         case 27:
2266         case 29:
2267         case 30:
2268                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2269                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2270                 break;
2271         default:
2272                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2273                 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2274                 break;
2275         }
2276
2277         lightrec_free_reg(reg_cache, rt);
2278 }
2279
2280 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2281                             const struct block *block, u16 offset,
2282                             u8 reg, u8 in_reg)
2283 {
2284         struct regcache *reg_cache = state->reg_cache;
2285         jit_state_t *_jit = block->_jit;
2286         u8 rt, tmp, tmp2, flags = 0;
2287
2288         _jit_name(block->_jit, __func__);
2289
2290         if (state->state->ops.cop2_notify) {
2291                 /* We must call cop2_notify, handle that in C. */
2292                 rec_mtc(state, block, offset);
2293                 return;
2294         }
2295
2296         if (reg == 31)
2297                 return;
2298
2299         if (reg == 30)
2300                 flags |= REG_EXT;
2301
2302         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
2303
2304         switch (reg) {
2305         case 15:
2306                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2307                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2308
2309                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2310                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2311
2312                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2313                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2314                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2315
2316                 lightrec_free_reg(reg_cache, tmp);
2317                 lightrec_free_reg(reg_cache, tmp2);
2318                 break;
2319         case 28:
2320                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2321
2322                 jit_lshi(tmp, rt, 7);
2323                 jit_andi(tmp, tmp, 0xf80);
2324                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2325
2326                 jit_lshi(tmp, rt, 2);
2327                 jit_andi(tmp, tmp, 0xf80);
2328                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2329
2330                 jit_rshi(tmp, rt, 3);
2331                 jit_andi(tmp, tmp, 0xf80);
2332                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2333
2334                 lightrec_free_reg(reg_cache, tmp);
2335                 break;
2336         case 30:
2337                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2338
2339                 /* if (rt < 0) rt = ~rt; */
2340                 jit_rshi(tmp, rt, 31);
2341                 jit_xorr(tmp, rt, tmp);
2342
2343                 /* Count leading zeros */
2344                 jit_clzr(tmp, tmp);
2345                 if (__WORDSIZE != 32)
2346                         jit_subi(tmp, tmp, __WORDSIZE - 32);
2347
2348                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp);
2349
2350                 lightrec_free_reg(reg_cache, tmp);
2351                 fallthrough;
2352         default:
2353                 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
2354                 break;
2355         }
2356
2357         lightrec_free_reg(reg_cache, rt);
2358 }
2359
2360 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2361                                const struct block *block, u16 offset)
2362 {
2363         const union code c = block->opcode_list[offset].c;
2364
2365         rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2366 }
2367
2368 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2369                                const struct block *block, u16 offset)
2370 {
2371         struct regcache *reg_cache = state->reg_cache;
2372         const union code c = block->opcode_list[offset].c;
2373         jit_state_t *_jit = block->_jit;
2374         u8 rt, tmp, tmp2;
2375
2376         _jit_name(block->_jit, __func__);
2377
2378         if (state->state->ops.cop2_notify) {
2379                 /* We must call cop2_notify, handle that in C. */
2380                 rec_mtc(state, block, offset);
2381                 return;
2382         }
2383
2384         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2385
2386         switch (c.r.rd) {
2387         case 4:
2388         case 12:
2389         case 20:
2390         case 26:
2391         case 27:
2392         case 29:
2393         case 30:
2394                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2395                 break;
2396         case 31:
2397                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2398                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2399
2400                 jit_andi(tmp, rt, 0x7f87e000);
2401                 jit_nei(tmp, tmp, 0);
2402                 jit_lshi(tmp, tmp, 31);
2403
2404                 jit_andi(tmp2, rt, 0x7ffff000);
2405                 jit_orr(tmp, tmp2, tmp);
2406
2407                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2408
2409                 lightrec_free_reg(reg_cache, tmp);
2410                 lightrec_free_reg(reg_cache, tmp2);
2411                 break;
2412
2413         default:
2414                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2415         }
2416
2417         lightrec_free_reg(reg_cache, rt);
2418 }
2419
2420 static void rec_cp0_RFE(struct lightrec_cstate *state,
2421                         const struct block *block, u16 offset)
2422 {
2423         struct regcache *reg_cache = state->reg_cache;
2424         jit_state_t *_jit = block->_jit;
2425         u8 status, tmp;
2426
2427         jit_name(__func__);
2428         jit_note(__FILE__, __LINE__);
2429
2430         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2431         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2432                    offsetof(struct lightrec_state, regs.cp0[12]));
2433
2434         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2435
2436         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2437         jit_rshi(tmp, status, 2);
2438         jit_andi(tmp, tmp, 0xf);
2439         jit_andi(status, status, ~0xful);
2440         jit_orr(status, status, tmp);
2441
2442         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2443                    offsetof(struct lightrec_state, regs.cp0[13]));
2444         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2445                    LIGHTREC_REG_STATE, status);
2446
2447         /* Exit dynarec in case there's a software interrupt.
2448          * exit_flags = !!(status & cause & 0x0300) & status; */
2449         jit_andr(tmp, tmp, status);
2450         jit_andi(tmp, tmp, 0x0300);
2451         jit_nei(tmp, tmp, 0);
2452         jit_andr(tmp, tmp, status);
2453         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2454                    LIGHTREC_REG_STATE, tmp);
2455
2456         lightrec_free_reg(reg_cache, status);
2457         lightrec_free_reg(reg_cache, tmp);
2458 }
2459
2460 static void rec_CP(struct lightrec_cstate *state,
2461                    const struct block *block, u16 offset)
2462 {
2463         union code c = block->opcode_list[offset].c;
2464         jit_state_t *_jit = block->_jit;
2465
2466         jit_name(__func__);
2467         jit_note(__FILE__, __LINE__);
2468
2469         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2470 }
2471
2472 static void rec_meta_MOV(struct lightrec_cstate *state,
2473                          const struct block *block, u16 offset)
2474 {
2475         struct regcache *reg_cache = state->reg_cache;
2476         const struct opcode *op = &block->opcode_list[offset];
2477         union code c = op->c;
2478         jit_state_t *_jit = block->_jit;
2479         bool unload_rd;
2480         u8 rs, rd;
2481
2482         _jit_name(block->_jit, __func__);
2483         jit_note(__FILE__, __LINE__);
2484
2485         unload_rd = OPT_EARLY_UNLOAD
2486                 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2487
2488         if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) {
2489                 /* The source register is not yet loaded - we can load its value
2490                  * from the register cache directly into the target register. */
2491                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2492
2493                 jit_ldxi_i(rd, LIGHTREC_REG_STATE,
2494                            offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2));
2495
2496                 lightrec_free_reg(reg_cache, rd);
2497         } else if (unload_rd) {
2498                 /* If the destination register will be unloaded right after the
2499                  * MOV meta-opcode, we don't actually need to write any host
2500                  * register - we can just store the source register directly to
2501                  * the register cache, at the offset corresponding to the
2502                  * destination register. */
2503                 lightrec_discard_reg_if_loaded(reg_cache, c.m.rd);
2504
2505                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2506
2507                 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
2508                            + (c.m.rd << 2), LIGHTREC_REG_STATE, rs);
2509
2510                 lightrec_free_reg(reg_cache, rs);
2511         } else {
2512                 if (c.m.rs)
2513                         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2514
2515                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2516
2517                 if (c.m.rs == 0) {
2518                         jit_movi(rd, 0);
2519                 } else {
2520                         jit_extr_i(rd, rs);
2521                         lightrec_free_reg(reg_cache, rs);
2522                 }
2523
2524                 lightrec_free_reg(reg_cache, rd);
2525         }
2526 }
2527
2528 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2529                                const struct block *block,
2530                                u16 offset)
2531 {
2532         struct regcache *reg_cache = state->reg_cache;
2533         union code c = block->opcode_list[offset].c;
2534         jit_state_t *_jit = block->_jit;
2535         u8 rs, rd;
2536
2537         _jit_name(block->_jit, __func__);
2538         jit_note(__FILE__, __LINE__);
2539
2540         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2541         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2542
2543         if (c.m.op == OP_META_EXTC)
2544                 jit_extr_c(rd, rs);
2545         else
2546                 jit_extr_s(rd, rs);
2547
2548         lightrec_free_reg(reg_cache, rs);
2549         lightrec_free_reg(reg_cache, rd);
2550 }
2551
2552 static void rec_meta_MULT2(struct lightrec_cstate *state,
2553                            const struct block *block,
2554                            u16 offset)
2555 {
2556         struct regcache *reg_cache = state->reg_cache;
2557         union code c = block->opcode_list[offset].c;
2558         jit_state_t *_jit = block->_jit;
2559         u8 reg_lo = get_mult_div_lo(c);
2560         u8 reg_hi = get_mult_div_hi(c);
2561         u32 flags = block->opcode_list[offset].flags;
2562         bool is_signed = c.i.op == OP_META_MULT2;
2563         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2564         unsigned int i;
2565
2566         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2567                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2568                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2569         }
2570
2571         _jit_name(block->_jit, __func__);
2572         jit_note(__FILE__, __LINE__);
2573
2574         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2575
2576         /*
2577          * We must handle the case where one of the output registers is our rs
2578          * input register. Thanksfully, computing LO/HI can be done in any
2579          * order. Here, we make sure that the computation that overwrites the
2580          * input register is always performed last.
2581          */
2582         for (i = 0; i < 2; i++) {
2583                 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2584                         lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2585
2586                         if (c.r.op < 32)
2587                                 jit_lshi(lo, rs, c.r.op);
2588                         else
2589                                 jit_movi(lo, 0);
2590
2591                         lightrec_free_reg(reg_cache, lo);
2592                         continue;
2593                 }
2594
2595                 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2596                         hi = lightrec_alloc_reg_out(reg_cache, _jit,
2597                                                     reg_hi, hiflags);
2598
2599                         if (c.r.op >= 32)
2600                                 jit_lshi(hi, rs, c.r.op - 32);
2601                         else if (is_signed)
2602                                 jit_rshi(hi, rs, 32 - c.r.op);
2603                         else
2604                                 jit_rshi_u(hi, rs, 32 - c.r.op);
2605
2606                         lightrec_free_reg(reg_cache, hi);
2607                 }
2608         }
2609
2610         lightrec_free_reg(reg_cache, rs);
2611
2612         _jit_name(block->_jit, __func__);
2613         jit_note(__FILE__, __LINE__);
2614 }
2615
2616 static void rec_meta_COM(struct lightrec_cstate *state,
2617                          const struct block *block, u16 offset)
2618 {
2619         struct regcache *reg_cache = state->reg_cache;
2620         union code c = block->opcode_list[offset].c;
2621         jit_state_t *_jit = block->_jit;
2622         u8 rd, rs, flags;
2623
2624         jit_note(__FILE__, __LINE__);
2625         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2626         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, 0);
2627
2628         flags = lightrec_get_reg_in_flags(reg_cache, rs);
2629
2630         lightrec_set_reg_out_flags(reg_cache, rd,
2631                                    flags & REG_EXT);
2632
2633         jit_comr(rd, rs);
2634
2635         lightrec_free_reg(reg_cache, rs);
2636         lightrec_free_reg(reg_cache, rd);
2637 }
2638
2639 static const lightrec_rec_func_t rec_standard[64] = {
2640         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2641         [OP_SPECIAL]            = rec_SPECIAL,
2642         [OP_REGIMM]             = rec_REGIMM,
2643         [OP_J]                  = rec_J,
2644         [OP_JAL]                = rec_JAL,
2645         [OP_BEQ]                = rec_BEQ,
2646         [OP_BNE]                = rec_BNE,
2647         [OP_BLEZ]               = rec_BLEZ,
2648         [OP_BGTZ]               = rec_BGTZ,
2649         [OP_ADDI]               = rec_ADDI,
2650         [OP_ADDIU]              = rec_ADDIU,
2651         [OP_SLTI]               = rec_SLTI,
2652         [OP_SLTIU]              = rec_SLTIU,
2653         [OP_ANDI]               = rec_ANDI,
2654         [OP_ORI]                = rec_ORI,
2655         [OP_XORI]               = rec_XORI,
2656         [OP_LUI]                = rec_LUI,
2657         [OP_CP0]                = rec_CP0,
2658         [OP_CP2]                = rec_CP2,
2659         [OP_LB]                 = rec_LB,
2660         [OP_LH]                 = rec_LH,
2661         [OP_LWL]                = rec_LWL,
2662         [OP_LW]                 = rec_LW,
2663         [OP_LBU]                = rec_LBU,
2664         [OP_LHU]                = rec_LHU,
2665         [OP_LWR]                = rec_LWR,
2666         [OP_SB]                 = rec_SB,
2667         [OP_SH]                 = rec_SH,
2668         [OP_SWL]                = rec_SWL,
2669         [OP_SW]                 = rec_SW,
2670         [OP_SWR]                = rec_SWR,
2671         [OP_LWC2]               = rec_LW,
2672         [OP_SWC2]               = rec_SW,
2673
2674         [OP_META]               = rec_META,
2675         [OP_META_MULT2]         = rec_meta_MULT2,
2676         [OP_META_MULTU2]        = rec_meta_MULT2,
2677 };
2678
2679 static const lightrec_rec_func_t rec_special[64] = {
2680         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2681         [OP_SPECIAL_SLL]        = rec_special_SLL,
2682         [OP_SPECIAL_SRL]        = rec_special_SRL,
2683         [OP_SPECIAL_SRA]        = rec_special_SRA,
2684         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2685         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2686         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2687         [OP_SPECIAL_JR]         = rec_special_JR,
2688         [OP_SPECIAL_JALR]       = rec_special_JALR,
2689         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2690         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2691         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2692         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2693         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2694         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2695         [OP_SPECIAL_MULT]       = rec_special_MULT,
2696         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2697         [OP_SPECIAL_DIV]        = rec_special_DIV,
2698         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2699         [OP_SPECIAL_ADD]        = rec_special_ADD,
2700         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2701         [OP_SPECIAL_SUB]        = rec_special_SUB,
2702         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2703         [OP_SPECIAL_AND]        = rec_special_AND,
2704         [OP_SPECIAL_OR]         = rec_special_OR,
2705         [OP_SPECIAL_XOR]        = rec_special_XOR,
2706         [OP_SPECIAL_NOR]        = rec_special_NOR,
2707         [OP_SPECIAL_SLT]        = rec_special_SLT,
2708         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2709 };
2710
2711 static const lightrec_rec_func_t rec_regimm[64] = {
2712         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2713         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2714         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2715         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2716         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2717 };
2718
2719 static const lightrec_rec_func_t rec_cp0[64] = {
2720         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2721         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2722         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2723         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2724         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2725         [OP_CP0_RFE]            = rec_cp0_RFE,
2726 };
2727
2728 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2729         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2730         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2731         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2732         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2733         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2734 };
2735
2736 static const lightrec_rec_func_t rec_meta[64] = {
2737         SET_DEFAULT_ELM(rec_meta, unknown_opcode),
2738         [OP_META_MOV]           = rec_meta_MOV,
2739         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2740         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2741         [OP_META_COM]           = rec_meta_COM,
2742 };
2743
2744 static void rec_SPECIAL(struct lightrec_cstate *state,
2745                         const struct block *block, u16 offset)
2746 {
2747         union code c = block->opcode_list[offset].c;
2748         lightrec_rec_func_t f = rec_special[c.r.op];
2749
2750         if (!HAS_DEFAULT_ELM && unlikely(!f))
2751                 unknown_opcode(state, block, offset);
2752         else
2753                 (*f)(state, block, offset);
2754 }
2755
2756 static void rec_REGIMM(struct lightrec_cstate *state,
2757                        const struct block *block, u16 offset)
2758 {
2759         union code c = block->opcode_list[offset].c;
2760         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2761
2762         if (!HAS_DEFAULT_ELM && unlikely(!f))
2763                 unknown_opcode(state, block, offset);
2764         else
2765                 (*f)(state, block, offset);
2766 }
2767
2768 static void rec_CP0(struct lightrec_cstate *state,
2769                     const struct block *block, u16 offset)
2770 {
2771         union code c = block->opcode_list[offset].c;
2772         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2773
2774         if (!HAS_DEFAULT_ELM && unlikely(!f))
2775                 rec_CP(state, block, offset);
2776         else
2777                 (*f)(state, block, offset);
2778 }
2779
2780 static void rec_CP2(struct lightrec_cstate *state,
2781                     const struct block *block, u16 offset)
2782 {
2783         union code c = block->opcode_list[offset].c;
2784
2785         if (c.r.op == OP_CP2_BASIC) {
2786                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2787
2788                 if (HAS_DEFAULT_ELM || likely(f)) {
2789                         (*f)(state, block, offset);
2790                         return;
2791                 }
2792         }
2793
2794         rec_CP(state, block, offset);
2795 }
2796
2797 static void rec_META(struct lightrec_cstate *state,
2798                      const struct block *block, u16 offset)
2799 {
2800         union code c = block->opcode_list[offset].c;
2801         lightrec_rec_func_t f = rec_meta[c.m.op];
2802
2803         if (!HAS_DEFAULT_ELM && unlikely(!f))
2804                 unknown_opcode(state, block, offset);
2805         else
2806                 (*f)(state, block, offset);
2807 }
2808
2809 void lightrec_rec_opcode(struct lightrec_cstate *state,
2810                          const struct block *block, u16 offset)
2811 {
2812         struct regcache *reg_cache = state->reg_cache;
2813         struct lightrec_branch_target *target;
2814         const struct opcode *op = &block->opcode_list[offset];
2815         jit_state_t *_jit = block->_jit;
2816         lightrec_rec_func_t f;
2817         u16 unload_offset;
2818
2819         if (op_flag_sync(op->flags)) {
2820                 if (state->cycles)
2821                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2822                 state->cycles = 0;
2823
2824                 lightrec_storeback_regs(reg_cache, _jit);
2825                 lightrec_regcache_reset(reg_cache);
2826
2827                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2828                 target = &state->targets[state->nb_targets++];
2829                 target->offset = offset;
2830                 target->label = jit_indirect();
2831         }
2832
2833         if (likely(op->opcode)) {
2834                 f = rec_standard[op->i.op];
2835
2836                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2837                         unknown_opcode(state, block, offset);
2838                 else
2839                         (*f)(state, block, offset);
2840         }
2841
2842         if (OPT_EARLY_UNLOAD) {
2843                 unload_offset = offset +
2844                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2845
2846                 lightrec_do_early_unload(state, block, unload_offset);
2847         }
2848
2849         state->no_load_delay = false;
2850 }