b8025aacd5809f10fabcb69f4f425f8ce404c065
[pcsx_rearmed.git] / deps / lightrec / emitter.c
1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 /*
3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
4  */
5
6 #include "blockcache.h"
7 #include "debug.h"
8 #include "disassembler.h"
9 #include "emitter.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
12 #include "regcache.h"
13
14 #include <stdbool.h>
15 #include <stddef.h>
16
17 #define LIGHTNING_UNALIGNED_32BIT 4
18
19 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
20
21 /* Forward declarations */
22 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
24 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
25 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
26 static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset);
27 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
28                             const struct block *block, u16 offset, u8 reg, u8 in_reg);
29 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
30                             const struct block *block, u16 offset,
31                             u8 reg, u8 out_reg);
32
33 static void
34 lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void))
35 {
36         /* Prevent jit_jmpi() from using our cycles register as a temporary */
37         jit_live(LIGHTREC_REG_CYCLE);
38
39         jit_patch_abs(jit_jmpi(), fn);
40 }
41
42 static void
43 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
44 {
45         lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func);
46 }
47
48 static void
49 lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit)
50 {
51         lightrec_jump_to_fn(_jit, state->state->ds_check_func);
52 }
53
54 static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
55                                u8 ra_reg, u32 pc, u32 link)
56 {
57         u8 link_reg;
58
59         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
60         lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
61         lightrec_free_reg(reg_cache, link_reg);
62 }
63
64 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
65                                        const struct block *block, u16 offset,
66                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
67                                        u32 link, bool update_cycles)
68 {
69         struct regcache *reg_cache = state->reg_cache;
70         jit_state_t *_jit = block->_jit;
71         const struct opcode *op = &block->opcode_list[offset],
72                             *ds = get_delay_slot(block->opcode_list, offset);
73         u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c);
74         bool has_ds = has_delay_slot(op->c);
75
76         jit_note(__FILE__, __LINE__);
77
78         if (link && ra_reg != reg_new_pc)
79                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
80
81         if (reg_new_pc < 0)
82                 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
83         else
84                 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
85
86         if (link && ra_reg == reg_new_pc) {
87                 /* Handle the special case: JALR $r0, $r0
88                  * In that case the target PC should be the old value of the
89                  * register. */
90                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
91         }
92
93         if (has_ds && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
94                 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
95
96                 /* Recompile the delay slot */
97                 if (ds->c.opcode)
98                         lightrec_rec_opcode(state, block, offset + 1);
99         }
100
101         /* Clean the remaining registers */
102         lightrec_clean_regs(reg_cache, _jit);
103
104         if (cycles && update_cycles) {
105                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
106                 pr_debug("EOB: %u cycles\n", cycles);
107         }
108
109         if (has_ds && op_flag_load_delay(ds->flags)
110             && opcode_is_load(ds->c) && !state->no_load_delay) {
111                 /* If the delay slot is a load opcode, its target register
112                  * will be written after the first opcode of the target is
113                  * executed. Handle this by jumping to a special section of
114                  * the dispatcher. It expects the loaded value to be in
115                  * REG_TEMP, and the target register number to be in JIT_V1.*/
116                 jit_movi(JIT_V1, ds->c.i.rt);
117
118                 lightrec_jump_to_ds_check(state, _jit);
119         } else {
120                 lightrec_jump_to_eob(state, _jit);
121         }
122
123         lightrec_regcache_reset(reg_cache);
124 }
125
126 void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state,
127                                        const struct block *block, u16 offset)
128 {
129         struct regcache *reg_cache = state->reg_cache;
130         jit_state_t *_jit = block->_jit;
131
132         lightrec_clean_regs(reg_cache, _jit);
133
134         /* Call the interpreter with the block's address in JIT_V1 and the
135          * PC (which might have an offset) in JIT_V0. */
136         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
137                           block->pc + (offset << 2));
138         if (lightrec_store_next_pc()) {
139               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
140                          LIGHTREC_REG_STATE, JIT_V0);
141         }
142
143         jit_movi(JIT_V1, (uintptr_t)block);
144
145         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
146         lightrec_jump_to_fn(_jit, state->state->interpreter_func);
147 }
148
149 static void lightrec_emit_eob(struct lightrec_cstate *state,
150                               const struct block *block, u16 offset)
151 {
152         struct regcache *reg_cache = state->reg_cache;
153         jit_state_t *_jit = block->_jit;
154
155         lightrec_clean_regs(reg_cache, _jit);
156
157         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
158                           block->pc + (offset << 2));
159         if (lightrec_store_next_pc()) {
160               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
161                          LIGHTREC_REG_STATE, JIT_V0);
162         }
163
164         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
165
166         lightrec_jump_to_eob(state, _jit);
167 }
168
169 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
170 {
171         union code c = block->opcode_list[offset].c;
172
173         _jit_name(block->_jit, __func__);
174         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
175 }
176
177 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
178 {
179         union code c = block->opcode_list[offset].c;
180
181         _jit_name(block->_jit, __func__);
182         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
183                                    get_branch_pc(block, offset, 2), true);
184 }
185
186 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
187 {
188         union code c = block->opcode_list[offset].c;
189
190         _jit_name(block->_jit, __func__);
191         lightrec_emit_end_of_block(state, block, offset, -1,
192                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
193                                    31, 0, true);
194 }
195
196 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
197 {
198         union code c = block->opcode_list[offset].c;
199
200         _jit_name(block->_jit, __func__);
201         lightrec_emit_end_of_block(state, block, offset, -1,
202                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
203                                    31, get_branch_pc(block, offset, 2), true);
204 }
205
206 static void lightrec_do_early_unload(struct lightrec_cstate *state,
207                                      const struct block *block, u16 offset)
208 {
209         struct regcache *reg_cache = state->reg_cache;
210         const struct opcode *op = &block->opcode_list[offset];
211         jit_state_t *_jit = block->_jit;
212         unsigned int i;
213         u8 reg;
214         struct {
215                 u8 reg, op;
216         } reg_ops[3] = {
217                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
218                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
219                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
220         };
221
222         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
223                 reg = reg_ops[i].reg;
224
225                 switch (reg_ops[i].op) {
226                 case LIGHTREC_REG_UNLOAD:
227                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
228                         break;
229
230                 case LIGHTREC_REG_DISCARD:
231                         lightrec_discard_reg_if_loaded(reg_cache, reg);
232                         break;
233
234                 case LIGHTREC_REG_CLEAN:
235                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
236                         break;
237                 default:
238                         break;
239                 };
240         }
241 }
242
243 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
244                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
245 {
246         struct regcache *reg_cache = state->reg_cache;
247         struct native_register *regs_backup;
248         jit_state_t *_jit = block->_jit;
249         struct lightrec_branch *branch;
250         const struct opcode *op = &block->opcode_list[offset],
251                             *ds = get_delay_slot(block->opcode_list, offset);
252         jit_node_t *addr;
253         bool is_forward = (s16)op->i.imm >= 0;
254         int op_cycles = lightrec_cycles_of_opcode(state->state, op->c);
255         u32 target_offset, cycles = state->cycles + op_cycles;
256         bool no_indirection = false;
257         u32 next_pc;
258         u8 rs, rt;
259
260         jit_note(__FILE__, __LINE__);
261
262         if (!op_flag_no_ds(op->flags))
263                 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
264
265         state->cycles = -op_cycles;
266
267         if (!unconditional) {
268                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
269                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
270                                                     _jit, op->i.rt, REG_EXT);
271
272                 /* Unload dead registers before evaluating the branch */
273                 if (OPT_EARLY_UNLOAD)
274                         lightrec_do_early_unload(state, block, offset);
275
276                 if (op_flag_local_branch(op->flags) &&
277                     (op_flag_no_ds(op->flags) || !ds->opcode) &&
278                     is_forward && !lightrec_has_dirty_regs(reg_cache))
279                         no_indirection = true;
280
281                 if (no_indirection)
282                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
283         }
284
285         if (cycles)
286                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
287
288         if (!unconditional) {
289                 /* Generate the branch opcode */
290                 if (!no_indirection)
291                         addr = jit_new_node_pww(code, NULL, rs, rt);
292
293                 lightrec_free_regs(reg_cache);
294                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
295         }
296
297         if (op_flag_local_branch(op->flags)) {
298                 /* Recompile the delay slot */
299                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
300                         /* Never handle load delays with local branches. */
301                         state->no_load_delay = true;
302                         lightrec_rec_opcode(state, block, offset + 1);
303                 }
304
305                 if (link)
306                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
307
308                 /* Clean remaining registers */
309                 lightrec_clean_regs(reg_cache, _jit);
310
311                 target_offset = offset + 1 + (s16)op->i.imm
312                         - !!op_flag_no_ds(op->flags);
313                 pr_debug("Adding local branch to offset 0x%x\n",
314                          target_offset << 2);
315                 branch = &state->local_branches[
316                         state->nb_local_branches++];
317
318                 branch->target = target_offset;
319
320                 if (no_indirection)
321                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
322                 else if (is_forward)
323                         branch->branch = jit_b();
324                 else
325                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
326         }
327
328         if (!op_flag_local_branch(op->flags) || !is_forward) {
329                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
330                 state->no_load_delay = op_flag_local_branch(op->flags);
331                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
332                                            31, link, false);
333         }
334
335         if (!unconditional) {
336                 if (!no_indirection)
337                         jit_patch(addr);
338
339                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
340
341                 if (bz && link)
342                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
343
344                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
345                         state->no_load_delay = true;
346                         lightrec_rec_opcode(state, block, offset + 1);
347                 }
348         }
349 }
350
351 static void rec_BNE(struct lightrec_cstate *state,
352                     const struct block *block, u16 offset)
353 {
354         union code c = block->opcode_list[offset].c;
355
356         _jit_name(block->_jit, __func__);
357
358         if (c.i.rt == 0)
359                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
360         else
361                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
362 }
363
364 static void rec_BEQ(struct lightrec_cstate *state,
365                     const struct block *block, u16 offset)
366 {
367         union code c = block->opcode_list[offset].c;
368
369         _jit_name(block->_jit, __func__);
370
371         if (c.i.rt == 0)
372                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
373         else
374                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
375 }
376
377 static void rec_BLEZ(struct lightrec_cstate *state,
378                      const struct block *block, u16 offset)
379 {
380         union code c = block->opcode_list[offset].c;
381
382         _jit_name(block->_jit, __func__);
383         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
384 }
385
386 static void rec_BGTZ(struct lightrec_cstate *state,
387                      const struct block *block, u16 offset)
388 {
389         _jit_name(block->_jit, __func__);
390         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
391 }
392
393 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
394                             const struct block *block, u16 offset)
395 {
396         _jit_name(block->_jit, __func__);
397         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
398 }
399
400 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
401                               const struct block *block, u16 offset)
402 {
403         _jit_name(block->_jit, __func__);
404         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
405               get_branch_pc(block, offset, 2), false, true);
406 }
407
408 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
409                             const struct block *block, u16 offset)
410 {
411         union code c = block->opcode_list[offset].c;
412
413         _jit_name(block->_jit, __func__);
414         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
415 }
416
417 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
418                               const struct block *block, u16 offset)
419 {
420         const struct opcode *op = &block->opcode_list[offset];
421         _jit_name(block->_jit, __func__);
422         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
423               get_branch_pc(block, offset, 2),
424               !op->i.rs, true);
425 }
426
427 static void rec_alloc_rs_rd(struct regcache *reg_cache,
428                             jit_state_t *_jit,
429                             const struct opcode *op,
430                             u8 rs, u8 rd,
431                             u8 in_flags, u8 out_flags,
432                             u8 *rs_out, u8 *rd_out)
433 {
434         bool unload, discard;
435         u32 unload_flags;
436
437         if (OPT_EARLY_UNLOAD) {
438                 unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags);
439                 unload = unload_flags == LIGHTREC_REG_UNLOAD;
440                 discard = unload_flags == LIGHTREC_REG_DISCARD;
441         }
442
443         if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) {
444                 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
445                 lightrec_remap_reg(reg_cache, _jit, rs, rd, discard);
446                 lightrec_set_reg_out_flags(reg_cache, rs, out_flags);
447                 rd = rs;
448         } else {
449                 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
450                 rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags);
451         }
452
453         *rs_out = rs;
454         *rd_out = rd;
455 }
456
457 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
458                         u16 offset, jit_code_t code, bool slti)
459 {
460         struct regcache *reg_cache = state->reg_cache;
461         union code c = block->opcode_list[offset].c;
462         jit_state_t *_jit = block->_jit;
463         u8 rs, rt, out_flags = REG_EXT;
464
465         if (slti)
466                 out_flags |= REG_ZEXT;
467
468         jit_note(__FILE__, __LINE__);
469
470         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
471                         c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt);
472
473         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
474
475         lightrec_free_reg(reg_cache, rs);
476         lightrec_free_reg(reg_cache, rt);
477 }
478
479 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
480                             u16 offset, jit_code_t code, bool out_ext)
481 {
482         struct regcache *reg_cache = state->reg_cache;
483         union code c = block->opcode_list[offset].c;
484         jit_state_t *_jit = block->_jit;
485         u8 rd, rt, rs;
486
487         jit_note(__FILE__, __LINE__);
488
489         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
490         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
491                         c.r.rs, c.r.rd, REG_EXT,
492                         out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd);
493
494         jit_new_node_www(code, rd, rs, rt);
495
496         lightrec_free_reg(reg_cache, rs);
497         lightrec_free_reg(reg_cache, rt);
498         lightrec_free_reg(reg_cache, rd);
499 }
500
501 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
502                            u16 offset, jit_code_t code)
503 {
504         struct regcache *reg_cache = state->reg_cache;
505         union code c = block->opcode_list[offset].c;
506         jit_state_t *_jit = block->_jit;
507         u8 rd, rt, rs, temp, flags = 0;
508
509         jit_note(__FILE__, __LINE__);
510
511         if (code == jit_code_rshr)
512                 flags = REG_EXT;
513         else if (code == jit_code_rshr_u)
514                 flags = REG_ZEXT;
515
516         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
517         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
518                         c.r.rt, c.r.rd, flags, flags, &rt, &rd);
519
520         if (rt != rd) {
521                 jit_andi(rd, rs, 0x1f);
522                 jit_new_node_www(code, rd, rt, rd);
523         } else {
524                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
525                 jit_andi(temp, rs, 0x1f);
526                 jit_new_node_www(code, rd, rt, temp);
527                 lightrec_free_reg(reg_cache, temp);
528         }
529
530         lightrec_free_reg(reg_cache, rs);
531         lightrec_free_reg(reg_cache, rt);
532         lightrec_free_reg(reg_cache, rd);
533 }
534
535 static void rec_movi(struct lightrec_cstate *state,
536                      const struct block *block, u16 offset)
537 {
538         struct regcache *reg_cache = state->reg_cache;
539         union code c = block->opcode_list[offset].c;
540         jit_state_t *_jit = block->_jit;
541         u16 flags = REG_EXT;
542         s32 value = (s32)(s16) c.i.imm;
543         u8 rt;
544
545         if (block->opcode_list[offset].flags & LIGHTREC_MOVI)
546                 value += (s32)((u32)state->movi_temp[c.i.rt] << 16);
547
548         if (value >= 0)
549                 flags |= REG_ZEXT;
550
551         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
552
553         jit_movi(rt, value);
554
555         lightrec_free_reg(reg_cache, rt);
556 }
557
558 static void rec_ADDIU(struct lightrec_cstate *state,
559                       const struct block *block, u16 offset)
560 {
561         const struct opcode *op = &block->opcode_list[offset];
562
563         _jit_name(block->_jit, __func__);
564
565         if (op->i.rs && !(op->flags & LIGHTREC_MOVI))
566                 rec_alu_imm(state, block, offset, jit_code_addi, false);
567         else
568                 rec_movi(state, block, offset);
569 }
570
571 static void rec_ADDI(struct lightrec_cstate *state,
572                      const struct block *block, u16 offset)
573 {
574         /* TODO: Handle the exception? */
575         _jit_name(block->_jit, __func__);
576         rec_ADDIU(state, block, offset);
577 }
578
579 static void rec_SLTIU(struct lightrec_cstate *state,
580                       const struct block *block, u16 offset)
581 {
582         _jit_name(block->_jit, __func__);
583         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
584 }
585
586 static void rec_SLTI(struct lightrec_cstate *state,
587                      const struct block *block, u16 offset)
588 {
589         _jit_name(block->_jit, __func__);
590         rec_alu_imm(state, block, offset, jit_code_lti, true);
591 }
592
593 static void rec_ANDI(struct lightrec_cstate *state,
594                      const struct block *block, u16 offset)
595 {
596         struct regcache *reg_cache = state->reg_cache;
597         union code c = block->opcode_list[offset].c;
598         jit_state_t *_jit = block->_jit;
599         u8 rs, rt;
600
601         _jit_name(block->_jit, __func__);
602         jit_note(__FILE__, __LINE__);
603
604         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
605                         c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt);
606
607         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
608          * casts to uint8_t / uint16_t. */
609         if (c.i.imm == 0xff)
610                 jit_extr_uc(rt, rs);
611         else if (c.i.imm == 0xffff)
612                 jit_extr_us(rt, rs);
613         else
614                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
615
616         lightrec_free_reg(reg_cache, rs);
617         lightrec_free_reg(reg_cache, rt);
618 }
619
620 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
621                            u16 offset, jit_code_t code)
622 {
623         struct regcache *reg_cache = state->reg_cache;
624         union code c = block->opcode_list[offset].c;
625         jit_state_t *_jit = block->_jit;
626         u8 rs, rt, flags;
627
628         jit_note(__FILE__, __LINE__);
629
630         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
631                         c.i.rs, c.i.rt, 0, 0, &rs, &rt);
632
633         flags = lightrec_get_reg_in_flags(reg_cache, rs);
634         lightrec_set_reg_out_flags(reg_cache, rt, flags);
635
636         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
637
638         lightrec_free_reg(reg_cache, rs);
639         lightrec_free_reg(reg_cache, rt);
640 }
641
642
643 static void rec_ORI(struct lightrec_cstate *state,
644                     const struct block *block, u16 offset)
645 {
646         const struct opcode *op = &block->opcode_list[offset];
647         struct regcache *reg_cache = state->reg_cache;
648         jit_state_t *_jit = block->_jit;
649         s32 val;
650         u8 rt;
651
652         _jit_name(_jit, __func__);
653
654         if (op->flags & LIGHTREC_MOVI) {
655                 rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT);
656
657                 val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm;
658                 jit_movi(rt, val);
659
660                 lightrec_free_reg(reg_cache, rt);
661         } else {
662                 rec_alu_or_xor(state, block, offset, jit_code_ori);
663         }
664 }
665
666 static void rec_XORI(struct lightrec_cstate *state,
667                      const struct block *block, u16 offset)
668 {
669         _jit_name(block->_jit, __func__);
670         rec_alu_or_xor(state, block, offset, jit_code_xori);
671 }
672
673 static void rec_LUI(struct lightrec_cstate *state,
674                     const struct block *block, u16 offset)
675 {
676         struct regcache *reg_cache = state->reg_cache;
677         union code c = block->opcode_list[offset].c;
678         jit_state_t *_jit = block->_jit;
679         u8 rt, flags = REG_EXT;
680
681         if (block->opcode_list[offset].flags & LIGHTREC_MOVI) {
682                 state->movi_temp[c.i.rt] = c.i.imm;
683                 return;
684         }
685
686         jit_name(__func__);
687         jit_note(__FILE__, __LINE__);
688
689         if (!(c.i.imm & BIT(15)))
690                 flags |= REG_ZEXT;
691
692         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
693
694         jit_movi(rt, (s32)(c.i.imm << 16));
695
696         lightrec_free_reg(reg_cache, rt);
697 }
698
699 static void rec_special_ADDU(struct lightrec_cstate *state,
700                              const struct block *block, u16 offset)
701 {
702         _jit_name(block->_jit, __func__);
703         rec_alu_special(state, block, offset, jit_code_addr, false);
704 }
705
706 static void rec_special_ADD(struct lightrec_cstate *state,
707                             const struct block *block, u16 offset)
708 {
709         /* TODO: Handle the exception? */
710         _jit_name(block->_jit, __func__);
711         rec_alu_special(state, block, offset, jit_code_addr, false);
712 }
713
714 static void rec_special_SUBU(struct lightrec_cstate *state,
715                              const struct block *block, u16 offset)
716 {
717         _jit_name(block->_jit, __func__);
718         rec_alu_special(state, block, offset, jit_code_subr, false);
719 }
720
721 static void rec_special_SUB(struct lightrec_cstate *state,
722                             const struct block *block, u16 offset)
723 {
724         /* TODO: Handle the exception? */
725         _jit_name(block->_jit, __func__);
726         rec_alu_special(state, block, offset, jit_code_subr, false);
727 }
728
729 static void rec_special_AND(struct lightrec_cstate *state,
730                             const struct block *block, u16 offset)
731 {
732         struct regcache *reg_cache = state->reg_cache;
733         union code c = block->opcode_list[offset].c;
734         jit_state_t *_jit = block->_jit;
735         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
736
737         _jit_name(block->_jit, __func__);
738         jit_note(__FILE__, __LINE__);
739
740         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
741         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
742                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
743
744         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
745         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
746
747         /* Z(rd) = Z(rs) | Z(rt) */
748         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
749
750         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
751         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
752             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
753             (REG_EXT & flags_rs & flags_rt))
754                 flags_rd |= REG_EXT;
755
756         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
757
758         jit_andr(rd, rs, rt);
759
760         lightrec_free_reg(reg_cache, rs);
761         lightrec_free_reg(reg_cache, rt);
762         lightrec_free_reg(reg_cache, rd);
763 }
764
765 static void rec_special_or_nor(struct lightrec_cstate *state,
766                                const struct block *block, u16 offset, bool nor)
767 {
768         struct regcache *reg_cache = state->reg_cache;
769         union code c = block->opcode_list[offset].c;
770         jit_state_t *_jit = block->_jit;
771         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
772
773         jit_note(__FILE__, __LINE__);
774
775         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
776         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
777                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
778
779         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
780         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
781
782         /* or: Z(rd) = Z(rs) & Z(rt)
783          * nor: Z(rd) = 0 */
784         if (!nor)
785                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
786
787         /* E(rd) = E(rs) & E(rt) */
788         if (REG_EXT & flags_rs & flags_rt)
789                 flags_rd |= REG_EXT;
790
791         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
792
793         jit_orr(rd, rs, rt);
794
795         if (nor)
796                 jit_comr(rd, rd);
797
798         lightrec_free_reg(reg_cache, rs);
799         lightrec_free_reg(reg_cache, rt);
800         lightrec_free_reg(reg_cache, rd);
801 }
802
803 static void rec_special_OR(struct lightrec_cstate *state,
804                            const struct block *block, u16 offset)
805 {
806         _jit_name(block->_jit, __func__);
807         rec_special_or_nor(state, block, offset, false);
808 }
809
810 static void rec_special_NOR(struct lightrec_cstate *state,
811                             const struct block *block, u16 offset)
812 {
813         _jit_name(block->_jit, __func__);
814         rec_special_or_nor(state, block, offset, true);
815 }
816
817 static void rec_special_XOR(struct lightrec_cstate *state,
818                             const struct block *block, u16 offset)
819 {
820         struct regcache *reg_cache = state->reg_cache;
821         union code c = block->opcode_list[offset].c;
822         jit_state_t *_jit = block->_jit;
823         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
824
825         _jit_name(block->_jit, __func__);
826
827         jit_note(__FILE__, __LINE__);
828
829         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
830         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
831                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
832
833         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
834         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
835
836         /* Z(rd) = Z(rs) & Z(rt) */
837         flags_rd = REG_ZEXT & flags_rs & flags_rt;
838
839         /* E(rd) = E(rs) & E(rt) */
840         flags_rd |= REG_EXT & flags_rs & flags_rt;
841
842         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
843
844         jit_xorr(rd, rs, rt);
845
846         lightrec_free_reg(reg_cache, rs);
847         lightrec_free_reg(reg_cache, rt);
848         lightrec_free_reg(reg_cache, rd);
849 }
850
851 static void rec_special_SLTU(struct lightrec_cstate *state,
852                              const struct block *block, u16 offset)
853 {
854         _jit_name(block->_jit, __func__);
855         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
856 }
857
858 static void rec_special_SLT(struct lightrec_cstate *state,
859                             const struct block *block, u16 offset)
860 {
861         _jit_name(block->_jit, __func__);
862         rec_alu_special(state, block, offset, jit_code_ltr, true);
863 }
864
865 static void rec_special_SLLV(struct lightrec_cstate *state,
866                              const struct block *block, u16 offset)
867 {
868         _jit_name(block->_jit, __func__);
869         rec_alu_shiftv(state, block, offset, jit_code_lshr);
870 }
871
872 static void rec_special_SRLV(struct lightrec_cstate *state,
873                              const struct block *block, u16 offset)
874 {
875         _jit_name(block->_jit, __func__);
876         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
877 }
878
879 static void rec_special_SRAV(struct lightrec_cstate *state,
880                              const struct block *block, u16 offset)
881 {
882         _jit_name(block->_jit, __func__);
883         rec_alu_shiftv(state, block, offset, jit_code_rshr);
884 }
885
886 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
887                           u16 offset, jit_code_t code)
888 {
889         struct regcache *reg_cache = state->reg_cache;
890         union code c = block->opcode_list[offset].c;
891         jit_state_t *_jit = block->_jit;
892         u8 rd, rt, flags = 0, out_flags = 0;
893
894         jit_note(__FILE__, __LINE__);
895
896         if (code == jit_code_rshi)
897                 flags = REG_EXT;
898         else if (code == jit_code_rshi_u)
899                 flags = REG_ZEXT;
900
901         /* Input reg is zero-extended, if we SRL at least by one bit, we know
902          * the output reg will be both zero-extended and sign-extended. */
903         out_flags = flags;
904         if (code == jit_code_rshi_u && c.r.imm)
905                 out_flags |= REG_EXT;
906
907         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
908                         c.r.rt, c.r.rd, flags, out_flags, &rt, &rd);
909
910         jit_new_node_www(code, rd, rt, c.r.imm);
911
912         lightrec_free_reg(reg_cache, rt);
913         lightrec_free_reg(reg_cache, rd);
914 }
915
916 static void rec_special_SLL(struct lightrec_cstate *state,
917                             const struct block *block, u16 offset)
918 {
919         _jit_name(block->_jit, __func__);
920         rec_alu_shift(state, block, offset, jit_code_lshi);
921 }
922
923 static void rec_special_SRL(struct lightrec_cstate *state,
924                             const struct block *block, u16 offset)
925 {
926         _jit_name(block->_jit, __func__);
927         rec_alu_shift(state, block, offset, jit_code_rshi_u);
928 }
929
930 static void rec_special_SRA(struct lightrec_cstate *state,
931                             const struct block *block, u16 offset)
932 {
933         _jit_name(block->_jit, __func__);
934         rec_alu_shift(state, block, offset, jit_code_rshi);
935 }
936
937 static void rec_alu_mult(struct lightrec_cstate *state,
938                          const struct block *block, u16 offset, bool is_signed)
939 {
940         struct regcache *reg_cache = state->reg_cache;
941         union code c = block->opcode_list[offset].c;
942         u32 flags = block->opcode_list[offset].flags;
943         u8 reg_lo = get_mult_div_lo(c);
944         u8 reg_hi = get_mult_div_hi(c);
945         jit_state_t *_jit = block->_jit;
946         u8 lo, hi, rs, rt, rflags = 0;
947         bool no_lo = op_flag_no_lo(flags);
948         bool no_hi = op_flag_no_hi(flags);
949
950         jit_note(__FILE__, __LINE__);
951
952         if (is_signed)
953                 rflags = REG_EXT;
954         else
955                 rflags = REG_ZEXT;
956
957         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
958         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
959
960         if (!no_lo)
961                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
962
963         if (!no_hi)
964                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
965
966         if (__WORDSIZE == 32) {
967                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
968                  * operation if the MULT was detected a 32-bit only. */
969                 if (no_lo) {
970                         if (is_signed)
971                                 jit_hmulr(hi, rs, rt);
972                         else
973                                 jit_hmulr_u(hi, rs, rt);
974                 } else if (no_hi) {
975                         jit_mulr(lo, rs, rt);
976                 } else if (is_signed) {
977                         jit_qmulr(lo, hi, rs, rt);
978                 } else {
979                         jit_qmulr_u(lo, hi, rs, rt);
980                 }
981         } else {
982                 /* On 64-bit systems, do a 64*64->64 bit operation. */
983                 if (no_lo) {
984                         jit_mulr(hi, rs, rt);
985                         jit_rshi(hi, hi, 32);
986                 } else {
987                         jit_mulr(lo, rs, rt);
988
989                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
990                         if (!no_hi)
991                                 jit_rshi(hi, lo, 32);
992                 }
993         }
994
995         lightrec_free_reg(reg_cache, rs);
996         lightrec_free_reg(reg_cache, rt);
997         if (!no_lo)
998                 lightrec_free_reg(reg_cache, lo);
999         if (!no_hi)
1000                 lightrec_free_reg(reg_cache, hi);
1001 }
1002
1003 static void rec_alu_div(struct lightrec_cstate *state,
1004                         const struct block *block, u16 offset, bool is_signed)
1005 {
1006         struct regcache *reg_cache = state->reg_cache;
1007         union code c = block->opcode_list[offset].c;
1008         u32 flags = block->opcode_list[offset].flags;
1009         bool no_check = op_flag_no_div_check(flags);
1010         u8 reg_lo = get_mult_div_lo(c);
1011         u8 reg_hi = get_mult_div_hi(c);
1012         jit_state_t *_jit = block->_jit;
1013         jit_node_t *branch, *to_end;
1014         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
1015
1016         jit_note(__FILE__, __LINE__);
1017
1018         if (is_signed)
1019                 rflags = REG_EXT;
1020         else
1021                 rflags = REG_ZEXT;
1022
1023         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
1024         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
1025
1026         if (!op_flag_no_lo(flags))
1027                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
1028
1029         if (!op_flag_no_hi(flags))
1030                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
1031
1032         /* Jump to special handler if dividing by zero  */
1033         if (!no_check)
1034                 branch = jit_beqi(rt, 0);
1035
1036         if (op_flag_no_lo(flags)) {
1037                 if (is_signed)
1038                         jit_remr(hi, rs, rt);
1039                 else
1040                         jit_remr_u(hi, rs, rt);
1041         } else if (op_flag_no_hi(flags)) {
1042                 if (is_signed)
1043                         jit_divr(lo, rs, rt);
1044                 else
1045                         jit_divr_u(lo, rs, rt);
1046         } else {
1047                 if (is_signed)
1048                         jit_qdivr(lo, hi, rs, rt);
1049                 else
1050                         jit_qdivr_u(lo, hi, rs, rt);
1051         }
1052
1053         if (!no_check) {
1054                 /* Jump above the div-by-zero handler */
1055                 to_end = jit_b();
1056
1057                 jit_patch(branch);
1058
1059                 if (!op_flag_no_lo(flags)) {
1060                         if (is_signed) {
1061                                 jit_ltr(lo, rs, rt);
1062                                 jit_lshi(lo, lo, 1);
1063                                 jit_subi(lo, lo, 1);
1064                         } else {
1065                                 jit_subi(lo, rt, 1);
1066                         }
1067                 }
1068
1069                 if (!op_flag_no_hi(flags))
1070                         jit_movr(hi, rs);
1071
1072                 jit_patch(to_end);
1073         }
1074
1075         lightrec_free_reg(reg_cache, rs);
1076         lightrec_free_reg(reg_cache, rt);
1077
1078         if (!op_flag_no_lo(flags))
1079                 lightrec_free_reg(reg_cache, lo);
1080
1081         if (!op_flag_no_hi(flags))
1082                 lightrec_free_reg(reg_cache, hi);
1083 }
1084
1085 static void rec_special_MULT(struct lightrec_cstate *state,
1086                              const struct block *block, u16 offset)
1087 {
1088         _jit_name(block->_jit, __func__);
1089         rec_alu_mult(state, block, offset, true);
1090 }
1091
1092 static void rec_special_MULTU(struct lightrec_cstate *state,
1093                               const struct block *block, u16 offset)
1094 {
1095         _jit_name(block->_jit, __func__);
1096         rec_alu_mult(state, block, offset, false);
1097 }
1098
1099 static void rec_special_DIV(struct lightrec_cstate *state,
1100                             const struct block *block, u16 offset)
1101 {
1102         _jit_name(block->_jit, __func__);
1103         rec_alu_div(state, block, offset, true);
1104 }
1105
1106 static void rec_special_DIVU(struct lightrec_cstate *state,
1107                              const struct block *block, u16 offset)
1108 {
1109         _jit_name(block->_jit, __func__);
1110         rec_alu_div(state, block, offset, false);
1111 }
1112
1113 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1114                              const struct block *block, u16 offset,
1115                              u8 dst, u8 src)
1116 {
1117         struct regcache *reg_cache = state->reg_cache;
1118         jit_state_t *_jit = block->_jit;
1119
1120         jit_note(__FILE__, __LINE__);
1121
1122         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
1123                         src, dst, 0, REG_EXT, &src, &dst);
1124
1125         jit_extr_i(dst, src);
1126
1127         lightrec_free_reg(reg_cache, src);
1128         lightrec_free_reg(reg_cache, dst);
1129 }
1130
1131 static void rec_special_MFHI(struct lightrec_cstate *state,
1132                              const struct block *block, u16 offset)
1133 {
1134         union code c = block->opcode_list[offset].c;
1135
1136         _jit_name(block->_jit, __func__);
1137         rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI);
1138 }
1139
1140 static void rec_special_MTHI(struct lightrec_cstate *state,
1141                              const struct block *block, u16 offset)
1142 {
1143         union code c = block->opcode_list[offset].c;
1144
1145         _jit_name(block->_jit, __func__);
1146         rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs);
1147 }
1148
1149 static void rec_special_MFLO(struct lightrec_cstate *state,
1150                              const struct block *block, u16 offset)
1151 {
1152         union code c = block->opcode_list[offset].c;
1153
1154         _jit_name(block->_jit, __func__);
1155         rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO);
1156 }
1157
1158 static void rec_special_MTLO(struct lightrec_cstate *state,
1159                              const struct block *block, u16 offset)
1160 {
1161         union code c = block->opcode_list[offset].c;
1162
1163         _jit_name(block->_jit, __func__);
1164         rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs);
1165 }
1166
1167 static void call_to_c_wrapper(struct lightrec_cstate *state,
1168                               const struct block *block, u32 arg,
1169                               enum c_wrappers wrapper)
1170 {
1171         struct regcache *reg_cache = state->reg_cache;
1172         jit_state_t *_jit = block->_jit;
1173         s8 tmp, tmp2;
1174
1175         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1176         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1177
1178         tmp = lightrec_get_reg_with_value(reg_cache,
1179                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1180         if (tmp < 0) {
1181                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1182                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1183                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1184
1185                 lightrec_temp_set_value(reg_cache, tmp,
1186                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1187         }
1188
1189         lightrec_free_reg(reg_cache, tmp2);
1190
1191 #ifdef __mips__
1192         /* On MIPS, register t9 is always used as the target register for JALR.
1193          * Therefore if it does not contain the target address we must
1194          * invalidate it. */
1195         if (tmp != _T9)
1196                 lightrec_unload_reg(reg_cache, _jit, _T9);
1197 #endif
1198
1199         jit_prepare();
1200         jit_pushargi(arg);
1201
1202         lightrec_regcache_mark_live(reg_cache, _jit);
1203         jit_callr(tmp);
1204
1205         lightrec_free_reg(reg_cache, tmp);
1206         lightrec_regcache_mark_live(reg_cache, _jit);
1207 }
1208
1209 static void rec_io(struct lightrec_cstate *state,
1210                    const struct block *block, u16 offset,
1211                    bool load_rt, bool read_rt)
1212 {
1213         struct regcache *reg_cache = state->reg_cache;
1214         jit_state_t *_jit = block->_jit;
1215         union code c = block->opcode_list[offset].c;
1216         u32 flags = block->opcode_list[offset].flags;
1217         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1218         u32 lut_entry;
1219         u8 zero;
1220
1221         jit_note(__FILE__, __LINE__);
1222
1223         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1224
1225         if (read_rt && likely(c.i.rt))
1226                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1227         else if (load_rt)
1228                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1229
1230         if (op_flag_load_delay(flags) && !state->no_load_delay) {
1231                 /* Clear state->in_delay_slot_n. This notifies the lightrec_rw
1232                  * wrapper that it should write the REG_TEMP register instead of
1233                  * the actual output register of the opcode. */
1234                 zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1235                 jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n),
1236                             LIGHTREC_REG_STATE, zero);
1237                 lightrec_free_reg(reg_cache, zero);
1238         }
1239
1240         if (is_tagged) {
1241                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1242         } else {
1243                 lut_entry = lightrec_get_lut_entry(block);
1244                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1245                                   C_WRAPPER_RW_GENERIC);
1246         }
1247 }
1248
1249 static u32 rec_ram_mask(const struct lightrec_state *state)
1250 {
1251         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1252 }
1253
1254 static u32 rec_io_mask(const struct lightrec_state *state)
1255 {
1256         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1257
1258         return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0);
1259 }
1260
1261 static void rec_store_memory(struct lightrec_cstate *cstate,
1262                              const struct block *block,
1263                              u16 offset, jit_code_t code,
1264                              jit_code_t swap_code,
1265                              uintptr_t addr_offset, u32 addr_mask,
1266                              bool invalidate)
1267 {
1268         const struct lightrec_state *state = cstate->state;
1269         struct regcache *reg_cache = cstate->reg_cache;
1270         struct opcode *op = &block->opcode_list[offset];
1271         jit_state_t *_jit = block->_jit;
1272         union code c = op->c;
1273         u8 rs, rt, tmp = 0, tmp2 = 0, tmp3, addr_reg, addr_reg2;
1274         s16 imm = (s16)c.i.imm;
1275         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1276         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1277         bool no_mask = op_flag_no_mask(op->flags);
1278         bool add_imm = c.i.imm &&
1279                 (c.i.op == OP_META_SWU
1280                  || (!state->mirrors_mapped && !no_mask) || (invalidate &&
1281                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1282         bool need_tmp = !no_mask || add_imm || invalidate;
1283         bool swc2 = c.i.op == OP_SWC2;
1284         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1285         s8 reg_imm;
1286
1287         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1288         if (need_tmp)
1289                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1290
1291         addr_reg = rs;
1292
1293         if (add_imm) {
1294                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1295                 lightrec_free_reg(reg_cache, rs);
1296                 addr_reg = tmp;
1297                 imm = 0;
1298         } else if (simm) {
1299                 lut_offt += simm;
1300         }
1301
1302         if (!no_mask) {
1303                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1304                                                              addr_mask);
1305
1306                 jit_andr(tmp, addr_reg, reg_imm);
1307                 addr_reg = tmp;
1308
1309                 lightrec_free_reg(reg_cache, reg_imm);
1310         }
1311
1312         if (addr_offset) {
1313                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1314                                                              addr_offset);
1315                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1316                 jit_addr(tmp2, addr_reg, reg_imm);
1317                 addr_reg2 = tmp2;
1318
1319                 lightrec_free_reg(reg_cache, reg_imm);
1320         } else {
1321                 addr_reg2 = addr_reg;
1322         }
1323
1324         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1325
1326         if (is_big_endian() && swap_code && in_reg) {
1327                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1328
1329                 jit_new_node_ww(swap_code, tmp3, rt);
1330
1331                 if (c.i.op == OP_META_SWU)
1332                         jit_unstr(addr_reg2, tmp3, LIGHTNING_UNALIGNED_32BIT);
1333                 else
1334                         jit_new_node_www(code, imm, addr_reg2, tmp3);
1335
1336                 lightrec_free_reg(reg_cache, tmp3);
1337         } else if (c.i.op == OP_META_SWU) {
1338                 jit_unstr(addr_reg2, rt, LIGHTNING_UNALIGNED_32BIT);
1339         } else {
1340                 jit_new_node_www(code, imm, addr_reg2, rt);
1341         }
1342
1343         lightrec_free_reg(reg_cache, rt);
1344
1345         if (invalidate) {
1346                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1347
1348                 if (c.i.op != OP_SW) {
1349                         jit_andi(tmp, addr_reg, ~3);
1350                         addr_reg = tmp;
1351                 }
1352
1353                 if (!lut_is_32bit(state)) {
1354                         jit_lshi(tmp, addr_reg, 1);
1355                         addr_reg = tmp;
1356                 }
1357
1358                 if (addr_reg == rs && c.i.rs == 0) {
1359                         addr_reg = LIGHTREC_REG_STATE;
1360                 } else {
1361                         jit_add_state(tmp, addr_reg);
1362                         addr_reg = tmp;
1363                 }
1364
1365                 if (lut_is_32bit(state))
1366                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1367                 else
1368                         jit_stxi(lut_offt, addr_reg, tmp3);
1369
1370                 lightrec_free_reg(reg_cache, tmp3);
1371         }
1372
1373         if (addr_offset)
1374                 lightrec_free_reg(reg_cache, tmp2);
1375         if (need_tmp)
1376                 lightrec_free_reg(reg_cache, tmp);
1377         lightrec_free_reg(reg_cache, rs);
1378 }
1379
1380 static void rec_store_ram(struct lightrec_cstate *cstate,
1381                           const struct block *block,
1382                           u16 offset, jit_code_t code,
1383                           jit_code_t swap_code, bool invalidate)
1384 {
1385         const struct lightrec_state *state = cstate->state;
1386
1387         _jit_note(block->_jit, __FILE__, __LINE__);
1388
1389         return rec_store_memory(cstate, block, offset, code, swap_code,
1390                                 state->offset_ram, rec_ram_mask(state),
1391                                 invalidate);
1392 }
1393
1394 static void rec_store_scratch(struct lightrec_cstate *cstate,
1395                               const struct block *block, u16 offset,
1396                               jit_code_t code, jit_code_t swap_code)
1397 {
1398         _jit_note(block->_jit, __FILE__, __LINE__);
1399
1400         return rec_store_memory(cstate, block, offset, code, swap_code,
1401                                 cstate->state->offset_scratch,
1402                                 0x1fffffff, false);
1403 }
1404
1405 static void rec_store_io(struct lightrec_cstate *cstate,
1406                          const struct block *block, u16 offset,
1407                          jit_code_t code, jit_code_t swap_code)
1408 {
1409         _jit_note(block->_jit, __FILE__, __LINE__);
1410
1411         return rec_store_memory(cstate, block, offset, code, swap_code,
1412                                 cstate->state->offset_io,
1413                                 rec_io_mask(cstate->state), false);
1414 }
1415
1416 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1417                                            const struct block *block,
1418                                            u16 offset, jit_code_t code,
1419                                            jit_code_t swap_code)
1420 {
1421         const struct lightrec_state *state = cstate->state;
1422         struct regcache *reg_cache = cstate->reg_cache;
1423         union code c = block->opcode_list[offset].c;
1424         jit_state_t *_jit = block->_jit;
1425         jit_node_t *to_not_ram, *to_end;
1426         bool swc2 = c.i.op == OP_SWC2;
1427         u8 tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt;
1428         u32 addr_mask;
1429         s32 reg_imm;
1430         s16 imm;
1431
1432         jit_note(__FILE__, __LINE__);
1433         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1434         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1435
1436         if (state->mirrors_mapped)
1437                 addr_mask = 0x1f800000 | (4 * RAM_SIZE - 1);
1438         else
1439                 addr_mask = 0x1f800000 | (RAM_SIZE - 1);
1440
1441         reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, addr_mask);
1442
1443         /* Convert to KUNSEG and avoid RAM mirrors */
1444         if ((c.i.op == OP_META_SWU || !state->mirrors_mapped) && c.i.imm) {
1445                 imm = 0;
1446                 jit_addi(tmp, rs, (s16)c.i.imm);
1447                 jit_andr(tmp, tmp, reg_imm);
1448         } else {
1449                 imm = (s16)c.i.imm;
1450                 jit_andr(tmp, rs, reg_imm);
1451         }
1452
1453         lightrec_free_reg(reg_cache, rs);
1454         lightrec_free_reg(reg_cache, reg_imm);
1455
1456         if (state->offset_ram != state->offset_scratch) {
1457                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1458
1459                 to_not_ram = jit_bmsi(tmp, BIT(28));
1460
1461                 jit_movi(tmp2, state->offset_ram);
1462
1463                 to_end = jit_b();
1464                 jit_patch(to_not_ram);
1465
1466                 jit_movi(tmp2, state->offset_scratch);
1467                 jit_patch(to_end);
1468         } else if (state->offset_ram) {
1469                 tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1470                                                           state->offset_ram);
1471         }
1472
1473         if (state->offset_ram || state->offset_scratch) {
1474                 jit_addr(tmp, tmp, tmp2);
1475                 lightrec_free_reg(reg_cache, tmp2);
1476         }
1477
1478         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1479
1480         if (is_big_endian() && swap_code && in_reg) {
1481                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1482
1483                 jit_new_node_ww(swap_code, tmp2, rt);
1484
1485                 if (c.i.op == OP_META_SWU)
1486                         jit_unstr(tmp, tmp2, LIGHTNING_UNALIGNED_32BIT);
1487                 else
1488                         jit_new_node_www(code, imm, tmp, tmp2);
1489
1490                 lightrec_free_reg(reg_cache, tmp2);
1491         } else if (c.i.op == OP_META_SWU) {
1492                 jit_unstr(tmp, rt, LIGHTNING_UNALIGNED_32BIT);
1493         } else {
1494                 jit_new_node_www(code, imm, tmp, rt);
1495         }
1496
1497         lightrec_free_reg(reg_cache, rt);
1498         lightrec_free_reg(reg_cache, tmp);
1499 }
1500
1501 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1502                              u16 offset, jit_code_t code, jit_code_t swap_code)
1503 {
1504         const struct lightrec_state *state = cstate->state;
1505         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1506         struct regcache *reg_cache = cstate->reg_cache;
1507         union code c = block->opcode_list[offset].c;
1508         jit_state_t *_jit = block->_jit;
1509         jit_node_t *to_not_ram, *to_end;
1510         bool swc2 = c.i.op == OP_SWC2;
1511         u8 tmp, tmp2, tmp3, masked_reg, rs, rt;
1512         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1513         u32 addr_mask = 0x1f800000 | (ram_size - 1);
1514         bool different_offsets = state->offset_ram != state->offset_scratch;
1515         s32 reg_imm;
1516
1517         jit_note(__FILE__, __LINE__);
1518
1519         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1520         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1521         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1522
1523         reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, addr_mask);
1524
1525         /* Convert to KUNSEG and avoid RAM mirrors */
1526         if (c.i.imm) {
1527                 jit_addi(tmp2, rs, (s16)c.i.imm);
1528                 jit_andr(tmp2, tmp2, reg_imm);
1529         } else {
1530                 jit_andr(tmp2, rs, reg_imm);
1531         }
1532
1533         lightrec_free_reg(reg_cache, rs);
1534         lightrec_free_reg(reg_cache, reg_imm);
1535         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1536
1537         if (different_offsets) {
1538                 to_not_ram = jit_bgti(tmp2, ram_size);
1539                 masked_reg = tmp2;
1540         } else {
1541                 jit_lti_u(tmp, tmp2, ram_size);
1542                 jit_movnr(tmp, tmp2, tmp);
1543                 masked_reg = tmp;
1544         }
1545
1546         /* Compute the offset to the code LUT */
1547         if (c.i.op == OP_SW)
1548                 jit_andi(tmp, masked_reg, RAM_SIZE - 1);
1549         else
1550                 jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3);
1551
1552         if (!lut_is_32bit(state))
1553                 jit_lshi(tmp, tmp, 1);
1554         jit_add_state(tmp, tmp);
1555
1556         /* Write NULL to the code LUT to invalidate any block that's there */
1557         if (lut_is_32bit(state))
1558                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1559         else
1560                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1561
1562         if (c.i.op == OP_META_SWU) {
1563                 /* With a SWU opcode, we might have touched the following 32-bit
1564                  * word, so invalidate it as well */
1565                 if (lut_is_32bit(state)) {
1566                         jit_stxi_i(offsetof(struct lightrec_state, code_lut) + 4,
1567                                    tmp, tmp3);
1568                 } else {
1569                         jit_stxi(offsetof(struct lightrec_state, code_lut)
1570                                  + sizeof(uintptr_t), tmp, tmp3);
1571                 }
1572         }
1573
1574         if (different_offsets) {
1575                 jit_movi(tmp, state->offset_ram);
1576
1577                 to_end = jit_b();
1578                 jit_patch(to_not_ram);
1579         }
1580
1581         if (state->offset_ram || state->offset_scratch)
1582                 jit_movi(tmp, state->offset_scratch);
1583
1584         if (different_offsets)
1585                 jit_patch(to_end);
1586
1587         if (state->offset_ram || state->offset_scratch)
1588                 jit_addr(tmp2, tmp2, tmp);
1589
1590         lightrec_free_reg(reg_cache, tmp);
1591         lightrec_free_reg(reg_cache, tmp3);
1592
1593         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1594
1595         if (is_big_endian() && swap_code && in_reg) {
1596                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1597
1598                 jit_new_node_ww(swap_code, tmp, rt);
1599
1600                 if (c.i.op == OP_META_SWU)
1601                         jit_unstr(tmp2, tmp, LIGHTNING_UNALIGNED_32BIT);
1602                 else
1603                         jit_new_node_www(code, 0, tmp2, tmp);
1604
1605                 lightrec_free_reg(reg_cache, tmp);
1606         } else if (c.i.op == OP_META_SWU) {
1607                 jit_unstr(tmp2, rt, LIGHTNING_UNALIGNED_32BIT);
1608         } else {
1609                 jit_new_node_www(code, 0, tmp2, rt);
1610         }
1611
1612         lightrec_free_reg(reg_cache, rt);
1613         lightrec_free_reg(reg_cache, tmp2);
1614 }
1615
1616 static void rec_store(struct lightrec_cstate *state,
1617                       const struct block *block, u16 offset,
1618                       jit_code_t code, jit_code_t swap_code)
1619 {
1620         u32 flags = block->opcode_list[offset].flags;
1621         u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1622         bool no_invalidate = op_flag_no_invalidate(flags) ||
1623                 (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY);
1624         union code c = block->opcode_list[offset].c;
1625         bool is_swc2 = c.i.op == OP_SWC2;
1626
1627         if (is_swc2) {
1628                 switch (mode) {
1629                 case LIGHTREC_IO_RAM:
1630                 case LIGHTREC_IO_SCRATCH:
1631                 case LIGHTREC_IO_DIRECT:
1632                 case LIGHTREC_IO_DIRECT_HW:
1633                         rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP);
1634                         break;
1635                 default:
1636                         break;
1637                 }
1638         }
1639
1640         switch (mode) {
1641         case LIGHTREC_IO_RAM:
1642                 rec_store_ram(state, block, offset, code,
1643                               swap_code, !no_invalidate);
1644                 break;
1645         case LIGHTREC_IO_SCRATCH:
1646                 rec_store_scratch(state, block, offset, code, swap_code);
1647                 break;
1648         case LIGHTREC_IO_DIRECT:
1649                 if (no_invalidate) {
1650                         rec_store_direct_no_invalidate(state, block, offset,
1651                                                        code, swap_code);
1652                 } else {
1653                         rec_store_direct(state, block, offset, code, swap_code);
1654                 }
1655                 break;
1656         case LIGHTREC_IO_DIRECT_HW:
1657                 rec_store_io(state, block, offset, code, swap_code);
1658                 break;
1659         default:
1660                 rec_io(state, block, offset, true, false);
1661                 return;
1662         }
1663
1664         if (is_swc2)
1665                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1666 }
1667
1668 static void rec_SB(struct lightrec_cstate *state,
1669                    const struct block *block, u16 offset)
1670 {
1671         _jit_name(block->_jit, __func__);
1672         rec_store(state, block, offset, jit_code_stxi_c, 0);
1673 }
1674
1675 static void rec_SH(struct lightrec_cstate *state,
1676                    const struct block *block, u16 offset)
1677 {
1678         _jit_name(block->_jit, __func__);
1679         rec_store(state, block, offset,
1680                   jit_code_stxi_s, jit_code_bswapr_us);
1681 }
1682
1683 static void rec_SW(struct lightrec_cstate *state,
1684                    const struct block *block, u16 offset)
1685
1686 {
1687         union code c = block->opcode_list[offset].c;
1688
1689         _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
1690         rec_store(state, block, offset,
1691                   jit_code_stxi_i, jit_code_bswapr_ui);
1692 }
1693
1694 static void rec_SWL(struct lightrec_cstate *state,
1695                     const struct block *block, u16 offset)
1696 {
1697         _jit_name(block->_jit, __func__);
1698         rec_io(state, block, offset, true, false);
1699 }
1700
1701 static void rec_SWR(struct lightrec_cstate *state,
1702                     const struct block *block, u16 offset)
1703 {
1704         _jit_name(block->_jit, __func__);
1705         rec_io(state, block, offset, true, false);
1706 }
1707
1708 static void rec_load_memory(struct lightrec_cstate *cstate,
1709                             const struct block *block, u16 offset,
1710                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1711                             uintptr_t addr_offset, u32 addr_mask)
1712 {
1713         struct regcache *reg_cache = cstate->reg_cache;
1714         struct opcode *op = &block->opcode_list[offset];
1715         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1716         jit_state_t *_jit = block->_jit;
1717         u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
1718         bool no_mask = op_flag_no_mask(op->flags);
1719         union code c = op->c;
1720         s8 reg_imm;
1721         s16 imm;
1722
1723         if (load_delay || c.i.op == OP_LWC2)
1724                 out_reg = REG_TEMP;
1725         else if (c.i.rt)
1726                 out_reg = c.i.rt;
1727         else
1728                 return;
1729
1730         if (is_unsigned)
1731                 flags |= REG_ZEXT;
1732
1733         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1734         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1735
1736         if ((op->i.op == OP_META_LWU && c.i.imm)
1737             || (!cstate->state->mirrors_mapped && c.i.imm && !no_mask)) {
1738                 jit_addi(rt, rs, (s16)c.i.imm);
1739                 addr_reg = rt;
1740                 imm = 0;
1741         } else {
1742                 addr_reg = rs;
1743                 imm = (s16)c.i.imm;
1744         }
1745
1746         if (op->i.op == OP_META_LWU)
1747                 imm = LIGHTNING_UNALIGNED_32BIT;
1748
1749         if (!no_mask) {
1750                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1751                                                              addr_mask);
1752
1753                 jit_andr(rt, addr_reg, reg_imm);
1754                 addr_reg = rt;
1755
1756                 lightrec_free_reg(reg_cache, reg_imm);
1757         }
1758
1759         if (addr_offset) {
1760                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1761                                                              addr_offset);
1762
1763                 jit_addr(rt, addr_reg, reg_imm);
1764                 addr_reg = rt;
1765
1766                 lightrec_free_reg(reg_cache, reg_imm);
1767         }
1768
1769         jit_new_node_www(code, rt, addr_reg, imm);
1770
1771         if (is_big_endian() && swap_code) {
1772                 jit_new_node_ww(swap_code, rt, rt);
1773
1774                 if (c.i.op == OP_LH)
1775                         jit_extr_s(rt, rt);
1776                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1777                         jit_extr_i(rt, rt);
1778         }
1779
1780         lightrec_free_reg(reg_cache, rs);
1781         lightrec_free_reg(reg_cache, rt);
1782 }
1783
1784 static void rec_load_ram(struct lightrec_cstate *cstate,
1785                          const struct block *block, u16 offset,
1786                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1787 {
1788         _jit_note(block->_jit, __FILE__, __LINE__);
1789
1790         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1791                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1792 }
1793
1794 static void rec_load_bios(struct lightrec_cstate *cstate,
1795                           const struct block *block, u16 offset,
1796                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1797 {
1798         _jit_note(block->_jit, __FILE__, __LINE__);
1799
1800         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1801                         cstate->state->offset_bios, 0x1fffffff);
1802 }
1803
1804 static void rec_load_scratch(struct lightrec_cstate *cstate,
1805                              const struct block *block, u16 offset,
1806                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1807 {
1808         _jit_note(block->_jit, __FILE__, __LINE__);
1809
1810         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1811                         cstate->state->offset_scratch, 0x1fffffff);
1812 }
1813
1814 static void rec_load_io(struct lightrec_cstate *cstate,
1815                         const struct block *block, u16 offset,
1816                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1817 {
1818         _jit_note(block->_jit, __FILE__, __LINE__);
1819
1820         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1821                         cstate->state->offset_io, rec_io_mask(cstate->state));
1822 }
1823
1824 static void rec_load_direct(struct lightrec_cstate *cstate,
1825                             const struct block *block, u16 offset,
1826                             jit_code_t code, jit_code_t swap_code,
1827                             bool is_unsigned)
1828 {
1829         const struct lightrec_state *state = cstate->state;
1830         struct regcache *reg_cache = cstate->reg_cache;
1831         struct opcode *op = &block->opcode_list[offset];
1832         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1833         jit_state_t *_jit = block->_jit;
1834         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1835         u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
1836         bool different_offsets = state->offset_bios != state->offset_scratch;
1837         union code c = op->c;
1838         s32 addr_mask;
1839         u32 reg_imm;
1840         s8 offt_reg;
1841         s16 imm;
1842
1843         if (load_delay || c.i.op == OP_LWC2)
1844                 out_reg = REG_TEMP;
1845         else if (c.i.rt)
1846                 out_reg = c.i.rt;
1847         else
1848                 return;
1849
1850         if (is_unsigned)
1851                 flags |= REG_ZEXT;
1852
1853         jit_note(__FILE__, __LINE__);
1854         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1855         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1856
1857         if ((state->offset_ram == state->offset_bios &&
1858             state->offset_ram == state->offset_scratch &&
1859             state->mirrors_mapped && c.i.op != OP_META_LWU)
1860             || !c.i.imm) {
1861                 addr_reg = rs;
1862                 imm = (s16)c.i.imm;
1863         } else {
1864                 jit_addi(rt, rs, (s16)c.i.imm);
1865                 addr_reg = rt;
1866                 imm = 0;
1867
1868                 if (c.i.rs != c.i.rt)
1869                         lightrec_free_reg(reg_cache, rs);
1870         }
1871
1872         if (op->i.op == OP_META_LWU)
1873                 imm = LIGHTNING_UNALIGNED_32BIT;
1874
1875         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1876
1877         if (state->offset_ram == state->offset_bios &&
1878             state->offset_ram == state->offset_scratch) {
1879                 if (!state->mirrors_mapped)
1880                         addr_mask = 0x1f800000 | (RAM_SIZE - 1);
1881                 else
1882                         addr_mask = 0x1fffffff;
1883
1884                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1885                                                              addr_mask);
1886                 if (!state->mirrors_mapped) {
1887                         jit_andi(tmp, addr_reg, BIT(28));
1888                         jit_rshi_u(tmp, tmp, 28 - 22);
1889                         jit_orr(tmp, tmp, reg_imm);
1890                         jit_andr(rt, addr_reg, tmp);
1891                 } else {
1892                         jit_andr(rt, addr_reg, reg_imm);
1893                 }
1894
1895                 lightrec_free_reg(reg_cache, reg_imm);
1896
1897                 if (state->offset_ram) {
1898                         offt_reg = lightrec_get_reg_with_value(reg_cache,
1899                                                                state->offset_ram);
1900                         if (offt_reg < 0) {
1901                                 jit_movi(tmp, state->offset_ram);
1902                                 lightrec_temp_set_value(reg_cache, tmp,
1903                                                         state->offset_ram);
1904                         } else {
1905                                 lightrec_free_reg(reg_cache, tmp);
1906                                 tmp = offt_reg;
1907                         }
1908                 }
1909         } else {
1910                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1911
1912                 /* Convert to KUNSEG and avoid RAM mirrors */
1913                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1914
1915                 if (state->offset_ram)
1916                         jit_movi(tmp, state->offset_ram);
1917
1918                 to_end = jit_b();
1919
1920                 jit_patch(to_not_ram);
1921
1922                 if (different_offsets)
1923                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1924
1925                 /* Convert to KUNSEG */
1926                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1927
1928                 jit_movi(tmp, state->offset_bios);
1929
1930                 if (different_offsets) {
1931                         to_end2 = jit_b();
1932
1933                         jit_patch(to_not_bios);
1934
1935                         /* Convert to KUNSEG */
1936                         jit_andi(rt, addr_reg, 0x1f800fff);
1937
1938                         if (state->offset_scratch)
1939                                 jit_movi(tmp, state->offset_scratch);
1940
1941                         jit_patch(to_end2);
1942                 }
1943
1944                 jit_patch(to_end);
1945         }
1946
1947         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1948                 jit_addr(rt, rt, tmp);
1949
1950         jit_new_node_www(code, rt, rt, imm);
1951
1952         if (is_big_endian() && swap_code) {
1953                 jit_new_node_ww(swap_code, rt, rt);
1954
1955                 if (c.i.op == OP_LH)
1956                         jit_extr_s(rt, rt);
1957                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1958                         jit_extr_i(rt, rt);
1959         }
1960
1961         lightrec_free_reg(reg_cache, addr_reg);
1962         lightrec_free_reg(reg_cache, rt);
1963         lightrec_free_reg(reg_cache, tmp);
1964 }
1965
1966 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1967                      u16 offset, jit_code_t code, jit_code_t swap_code,
1968                      bool is_unsigned)
1969 {
1970         const struct opcode *op = &block->opcode_list[offset];
1971         u32 flags = op->flags;
1972
1973         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1974         case LIGHTREC_IO_RAM:
1975                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1976                 break;
1977         case LIGHTREC_IO_BIOS:
1978                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1979                 break;
1980         case LIGHTREC_IO_SCRATCH:
1981                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1982                 break;
1983         case LIGHTREC_IO_DIRECT_HW:
1984                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1985                 break;
1986         case LIGHTREC_IO_DIRECT:
1987                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1988                 break;
1989         default:
1990                 rec_io(state, block, offset, false, true);
1991                 return;
1992         }
1993
1994         if (op->i.op == OP_LWC2) {
1995                 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP);
1996                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1997         }
1998 }
1999
2000 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
2001 {
2002         _jit_name(block->_jit, __func__);
2003         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
2004 }
2005
2006 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
2007 {
2008         _jit_name(block->_jit, __func__);
2009         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
2010 }
2011
2012 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
2013 {
2014         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
2015
2016         _jit_name(block->_jit, __func__);
2017         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
2018 }
2019
2020 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
2021 {
2022         _jit_name(block->_jit, __func__);
2023         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
2024 }
2025
2026 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
2027 {
2028         _jit_name(block->_jit, __func__);
2029         rec_io(state, block, offset, true, true);
2030 }
2031
2032 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
2033 {
2034         _jit_name(block->_jit, __func__);
2035         rec_io(state, block, offset, true, true);
2036 }
2037
2038 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
2039 {
2040         union code c = block->opcode_list[offset].c;
2041         jit_code_t code;
2042
2043         if (is_big_endian() && __WORDSIZE == 64)
2044                 code = jit_code_ldxi_ui;
2045         else
2046                 code = jit_code_ldxi_i;
2047
2048         _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
2049         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2050 }
2051
2052 static void rec_exit_early(struct lightrec_cstate *state,
2053                            const struct block *block, u16 offset,
2054                            u32 exit_code, u32 pc)
2055 {
2056         struct regcache *reg_cache = state->reg_cache;
2057         jit_state_t *_jit = block->_jit;
2058         u8 tmp;
2059
2060         _jit_note(block->_jit, __FILE__, __LINE__);
2061
2062         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2063
2064         jit_movi(tmp, exit_code);
2065         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2066                    LIGHTREC_REG_STATE, tmp);
2067
2068         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2069                    offsetof(struct lightrec_state, target_cycle));
2070         jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE);
2071         jit_movi(LIGHTREC_REG_CYCLE, 0);
2072         jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2073                    LIGHTREC_REG_STATE, tmp);
2074         jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2075                    LIGHTREC_REG_STATE, tmp);
2076
2077         lightrec_free_reg(reg_cache, tmp);
2078
2079         lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true);
2080 }
2081
2082 static void rec_special_SYSCALL(struct lightrec_cstate *state,
2083                                 const struct block *block, u16 offset)
2084 {
2085         _jit_name(block->_jit, __func__);
2086
2087         /* TODO: the return address should be "pc - 4" if we're a delay slot */
2088         rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL,
2089                        get_ds_pc(block, offset, 0));
2090 }
2091
2092 static void rec_special_BREAK(struct lightrec_cstate *state,
2093                               const struct block *block, u16 offset)
2094 {
2095         _jit_name(block->_jit, __func__);
2096         rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK,
2097                        get_ds_pc(block, offset, 0));
2098 }
2099
2100 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2101 {
2102         struct regcache *reg_cache = state->reg_cache;
2103         union code c = block->opcode_list[offset].c;
2104         jit_state_t *_jit = block->_jit;
2105
2106         jit_note(__FILE__, __LINE__);
2107
2108         if (c.i.op != OP_SWC2)
2109                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
2110
2111         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
2112 }
2113
2114 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2115 {
2116         struct regcache *reg_cache = state->reg_cache;
2117         union code c = block->opcode_list[offset].c;
2118         jit_state_t *_jit = block->_jit;
2119
2120         jit_note(__FILE__, __LINE__);
2121         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
2122         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
2123         lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false);
2124
2125         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
2126
2127         if (c.i.op == OP_CP0 &&
2128             !op_flag_no_ds(block->opcode_list[offset].flags) &&
2129             (c.r.rd == 12 || c.r.rd == 13))
2130                 lightrec_emit_end_of_block(state, block, offset, -1,
2131                                            get_ds_pc(block, offset, 1),
2132                                            0, 0, true);
2133 }
2134
2135 static void
2136 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
2137 {
2138         struct regcache *reg_cache = state->reg_cache;
2139         union code c = block->opcode_list[offset].c;
2140         jit_state_t *_jit = block->_jit;
2141         u8 rt;
2142
2143         jit_note(__FILE__, __LINE__);
2144
2145         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2146
2147         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
2148                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
2149
2150         lightrec_free_reg(reg_cache, rt);
2151 }
2152
2153 static bool block_uses_icache(const struct lightrec_cstate *state,
2154                               const struct block *block)
2155 {
2156         const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM];
2157         u32 pc = kunseg(block->pc);
2158
2159         if (pc < map->pc || pc >= map->pc + map->length)
2160                 return false;
2161
2162         return (block->pc >> 28) < 0xa;
2163 }
2164
2165 static void
2166 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
2167 {
2168         struct regcache *reg_cache = state->reg_cache;
2169         const union code c = block->opcode_list[offset].c;
2170         jit_state_t *_jit = block->_jit;
2171         u8 rt, tmp = 0, tmp2, status;
2172         jit_node_t *to_end;
2173
2174         jit_note(__FILE__, __LINE__);
2175
2176         switch(c.r.rd) {
2177         case 1:
2178         case 4:
2179         case 8:
2180         case 14:
2181         case 15:
2182                 /* Those registers are read-only */
2183                 return;
2184         default:
2185                 break;
2186         }
2187
2188         if (!block_uses_icache(state, block) && c.r.rd == 12) {
2189                 /* If we are not running code from the RAM through kuseg or
2190                  * kseg0, handle writes to the Status register in C; as the
2191                  * code may toggle bit 16 which isolates the cache. Code
2192                  * running from kuseg or kseg0 in RAM cannot do that. */
2193                 rec_mtc(state, block, offset);
2194                 return;
2195         }
2196
2197         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
2198
2199         if (c.r.rd != 13) {
2200                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
2201                            LIGHTREC_REG_STATE, rt);
2202         }
2203
2204         if (c.r.rd == 12 || c.r.rd == 13) {
2205                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2206                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2207                            offsetof(struct lightrec_state, regs.cp0[13]));
2208
2209                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2210         }
2211
2212         if (c.r.rd == 12) {
2213                 status = rt;
2214         } else if (c.r.rd == 13) {
2215                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
2216                 jit_andi(tmp2, rt, 0x0300);
2217                 jit_ori(tmp, tmp, 0x0300);
2218                 jit_xori(tmp, tmp, 0x0300);
2219                 jit_orr(tmp, tmp, tmp2);
2220                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2221                            offsetof(struct lightrec_state, regs.cp0[12]));
2222                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
2223                            LIGHTREC_REG_STATE, tmp);
2224                 status = tmp2;
2225         }
2226
2227         if (c.r.rd == 12 || c.r.rd == 13) {
2228                 /* Exit dynarec in case there's a software interrupt.
2229                  * exit_flags = !!(status & tmp & 0x0300) & status; */
2230                 jit_andr(tmp, tmp, status);
2231                 jit_andi(tmp, tmp, 0x0300);
2232                 jit_nei(tmp, tmp, 0);
2233                 jit_andr(tmp, tmp, status);
2234         }
2235
2236         if (c.r.rd == 12) {
2237                 /* Exit dynarec in case we unmask a hardware interrupt.
2238                  * exit_flags = !(~status & 0x401) */
2239
2240                 jit_comr(tmp2, status);
2241                 jit_andi(tmp2, tmp2, 0x401);
2242                 jit_eqi(tmp2, tmp2, 0);
2243                 jit_orr(tmp, tmp, tmp2);
2244         }
2245
2246         lightrec_free_reg(reg_cache, rt);
2247
2248         if (c.r.rd == 12 || c.r.rd == 13) {
2249                 to_end = jit_beqi(tmp, 0);
2250
2251                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2252                            offsetof(struct lightrec_state, target_cycle));
2253                 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2254                 jit_movi(LIGHTREC_REG_CYCLE, 0);
2255                 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2256                            LIGHTREC_REG_STATE, tmp2);
2257                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2258                            LIGHTREC_REG_STATE, tmp2);
2259
2260
2261                 jit_patch(to_end);
2262         }
2263
2264         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
2265             (c.r.rd == 12 || c.r.rd == 13)) {
2266                 state->cycles += lightrec_cycles_of_opcode(state->state, c);
2267                 lightrec_emit_eob(state, block, offset + 1);
2268         }
2269 }
2270
2271 static void rec_cp0_MFC0(struct lightrec_cstate *state,
2272                          const struct block *block, u16 offset)
2273 {
2274         _jit_name(block->_jit, __func__);
2275         rec_mfc0(state, block, offset);
2276 }
2277
2278 static void rec_cp0_CFC0(struct lightrec_cstate *state,
2279                          const struct block *block, u16 offset)
2280 {
2281         _jit_name(block->_jit, __func__);
2282         rec_mfc0(state, block, offset);
2283 }
2284
2285 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2286                          const struct block *block, u16 offset)
2287 {
2288         _jit_name(block->_jit, __func__);
2289         rec_mtc0(state, block, offset);
2290 }
2291
2292 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2293                          const struct block *block, u16 offset)
2294 {
2295         _jit_name(block->_jit, __func__);
2296         rec_mtc0(state, block, offset);
2297 }
2298
2299 static unsigned int cp2d_i_offset(u8 reg)
2300 {
2301         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2302 }
2303
2304 static unsigned int cp2d_s_offset(u8 reg)
2305 {
2306         return cp2d_i_offset(reg) + is_big_endian() * 2;
2307 }
2308
2309 static unsigned int cp2c_i_offset(u8 reg)
2310 {
2311         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2312 }
2313
2314 static unsigned int cp2c_s_offset(u8 reg)
2315 {
2316         return cp2c_i_offset(reg) + is_big_endian() * 2;
2317 }
2318
2319 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2320                             const struct block *block, u16 offset,
2321                             u8 reg, u8 out_reg)
2322 {
2323         struct regcache *reg_cache = state->reg_cache;
2324         jit_state_t *_jit = block->_jit;
2325         const u32 zext_regs = 0x300f0080;
2326         u8 rt, tmp, tmp2, tmp3, out, flags;
2327         unsigned int i;
2328
2329         _jit_name(block->_jit, __func__);
2330
2331         if (state->state->ops.cop2_notify) {
2332                 /* We must call cop2_notify, handle that in C. */
2333                 rec_mfc(state, block, offset);
2334                 return;
2335         }
2336
2337         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2338         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2339
2340         if (reg == 15)
2341                 reg = 14;
2342
2343         switch (reg) {
2344         case 1:
2345         case 3:
2346         case 5:
2347         case 8:
2348         case 9:
2349         case 10:
2350         case 11:
2351                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2352                 break;
2353         case 7:
2354         case 16:
2355         case 17:
2356         case 18:
2357         case 19:
2358                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2359                 break;
2360         case 28:
2361         case 29:
2362                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2363                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2364                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2365
2366                 for (i = 0; i < 3; i++) {
2367                         out = i == 0 ? rt : tmp;
2368
2369                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2370                         jit_movi(tmp2, 0x1f);
2371                         jit_rshi(out, tmp, 7);
2372
2373                         jit_ltr(tmp3, tmp2, out);
2374                         jit_movnr(out, tmp2, tmp3);
2375
2376                         jit_gei(tmp2, out, 0);
2377                         jit_movzr(out, tmp2, tmp2);
2378
2379                         if (i > 0) {
2380                                 jit_lshi(tmp, tmp, 5 * i);
2381                                 jit_orr(rt, rt, tmp);
2382                         }
2383                 }
2384
2385
2386                 lightrec_free_reg(reg_cache, tmp);
2387                 lightrec_free_reg(reg_cache, tmp2);
2388                 lightrec_free_reg(reg_cache, tmp3);
2389                 break;
2390         default:
2391                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2392                 break;
2393         }
2394
2395         lightrec_free_reg(reg_cache, rt);
2396 }
2397
2398 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2399                                const struct block *block, u16 offset)
2400 {
2401         const union code c = block->opcode_list[offset].c;
2402
2403         rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2404 }
2405
2406 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2407                                const struct block *block, u16 offset)
2408 {
2409         struct regcache *reg_cache = state->reg_cache;
2410         const union code c = block->opcode_list[offset].c;
2411         jit_state_t *_jit = block->_jit;
2412         u8 rt;
2413
2414         _jit_name(block->_jit, __func__);
2415
2416         if (state->state->ops.cop2_notify) {
2417                 /* We must call cop2_notify, handle that in C. */
2418                 rec_mfc(state, block, offset);
2419                 return;
2420         }
2421
2422         switch (c.r.rd) {
2423         case 4:
2424         case 12:
2425         case 20:
2426         case 26:
2427         case 27:
2428         case 29:
2429         case 30:
2430                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2431                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2432                 break;
2433         default:
2434                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2435                 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2436                 break;
2437         }
2438
2439         lightrec_free_reg(reg_cache, rt);
2440 }
2441
2442 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2443                             const struct block *block, u16 offset,
2444                             u8 reg, u8 in_reg)
2445 {
2446         struct regcache *reg_cache = state->reg_cache;
2447         jit_state_t *_jit = block->_jit;
2448         u8 rt, tmp, tmp2, flags = 0;
2449
2450         _jit_name(block->_jit, __func__);
2451
2452         if (state->state->ops.cop2_notify) {
2453                 /* We must call cop2_notify, handle that in C. */
2454                 rec_mtc(state, block, offset);
2455                 return;
2456         }
2457
2458         if (reg == 31)
2459                 return;
2460
2461         if (reg == 30)
2462                 flags |= REG_EXT;
2463
2464         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
2465
2466         switch (reg) {
2467         case 15:
2468                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2469                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2470
2471                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2472                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2473
2474                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2475                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2476                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2477
2478                 lightrec_free_reg(reg_cache, tmp);
2479                 lightrec_free_reg(reg_cache, tmp2);
2480                 break;
2481         case 28:
2482                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2483
2484                 jit_lshi(tmp, rt, 7);
2485                 jit_andi(tmp, tmp, 0xf80);
2486                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2487
2488                 jit_lshi(tmp, rt, 2);
2489                 jit_andi(tmp, tmp, 0xf80);
2490                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2491
2492                 jit_rshi(tmp, rt, 3);
2493                 jit_andi(tmp, tmp, 0xf80);
2494                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2495
2496                 lightrec_free_reg(reg_cache, tmp);
2497                 break;
2498         case 30:
2499                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2500
2501                 /* if (rt < 0) rt = ~rt; */
2502                 jit_rshi(tmp, rt, 31);
2503                 jit_xorr(tmp, rt, tmp);
2504
2505                 /* Count leading zeros */
2506                 jit_clzr(tmp, tmp);
2507                 if (__WORDSIZE != 32)
2508                         jit_subi(tmp, tmp, __WORDSIZE - 32);
2509
2510                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp);
2511
2512                 lightrec_free_reg(reg_cache, tmp);
2513                 fallthrough;
2514         default:
2515                 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
2516                 break;
2517         }
2518
2519         lightrec_free_reg(reg_cache, rt);
2520 }
2521
2522 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2523                                const struct block *block, u16 offset)
2524 {
2525         const union code c = block->opcode_list[offset].c;
2526
2527         rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2528 }
2529
2530 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2531                                const struct block *block, u16 offset)
2532 {
2533         struct regcache *reg_cache = state->reg_cache;
2534         const union code c = block->opcode_list[offset].c;
2535         jit_state_t *_jit = block->_jit;
2536         u8 rt, tmp, tmp2;
2537
2538         _jit_name(block->_jit, __func__);
2539
2540         if (state->state->ops.cop2_notify) {
2541                 /* We must call cop2_notify, handle that in C. */
2542                 rec_mtc(state, block, offset);
2543                 return;
2544         }
2545
2546         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2547
2548         switch (c.r.rd) {
2549         case 4:
2550         case 12:
2551         case 20:
2552         case 26:
2553         case 27:
2554         case 29:
2555         case 30:
2556                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2557                 break;
2558         case 31:
2559                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2560                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2561
2562                 jit_andi(tmp, rt, 0x7f87e000);
2563                 jit_nei(tmp, tmp, 0);
2564                 jit_lshi(tmp, tmp, 31);
2565
2566                 jit_andi(tmp2, rt, 0x7ffff000);
2567                 jit_orr(tmp, tmp2, tmp);
2568
2569                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2570
2571                 lightrec_free_reg(reg_cache, tmp);
2572                 lightrec_free_reg(reg_cache, tmp2);
2573                 break;
2574
2575         default:
2576                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2577         }
2578
2579         lightrec_free_reg(reg_cache, rt);
2580 }
2581
2582 static void rec_cp0_RFE(struct lightrec_cstate *state,
2583                         const struct block *block, u16 offset)
2584 {
2585         struct regcache *reg_cache = state->reg_cache;
2586         jit_state_t *_jit = block->_jit;
2587         u8 status, tmp;
2588
2589         jit_name(__func__);
2590         jit_note(__FILE__, __LINE__);
2591
2592         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2593         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2594                    offsetof(struct lightrec_state, regs.cp0[12]));
2595
2596         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2597
2598         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2599         jit_rshi(tmp, status, 2);
2600         jit_andi(tmp, tmp, 0xf);
2601         jit_andi(status, status, ~0xful);
2602         jit_orr(status, status, tmp);
2603
2604         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2605                    offsetof(struct lightrec_state, regs.cp0[13]));
2606         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2607                    LIGHTREC_REG_STATE, status);
2608
2609         /* Exit dynarec in case there's a software interrupt.
2610          * exit_flags = !!(status & cause & 0x0300) & status; */
2611         jit_andr(tmp, tmp, status);
2612         jit_andi(tmp, tmp, 0x0300);
2613         jit_nei(tmp, tmp, 0);
2614         jit_andr(tmp, tmp, status);
2615         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2616                    LIGHTREC_REG_STATE, tmp);
2617
2618         lightrec_free_reg(reg_cache, status);
2619         lightrec_free_reg(reg_cache, tmp);
2620 }
2621
2622 static void rec_CP(struct lightrec_cstate *state,
2623                    const struct block *block, u16 offset)
2624 {
2625         union code c = block->opcode_list[offset].c;
2626         jit_state_t *_jit = block->_jit;
2627
2628         jit_name(__func__);
2629         jit_note(__FILE__, __LINE__);
2630
2631         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2632 }
2633
2634 static void rec_meta_MOV(struct lightrec_cstate *state,
2635                          const struct block *block, u16 offset)
2636 {
2637         struct regcache *reg_cache = state->reg_cache;
2638         const struct opcode *op = &block->opcode_list[offset];
2639         union code c = op->c;
2640         jit_state_t *_jit = block->_jit;
2641         bool unload_rd;
2642         bool unload_rs, discard_rs;
2643         u8 rs, rd;
2644
2645         _jit_name(block->_jit, __func__);
2646         jit_note(__FILE__, __LINE__);
2647
2648         unload_rs = OPT_EARLY_UNLOAD
2649                 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD;
2650         discard_rs = OPT_EARLY_UNLOAD
2651                 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD;
2652
2653         if ((unload_rs || discard_rs) && c.m.rs) {
2654                 /* If the source register is going to be unloaded or discarded,
2655                  * then we can simply mark its host register as now pointing to
2656                  * the destination register. */
2657                 pr_debug("Remap %s to %s at offset 0x%x\n",
2658                          lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd),
2659                          offset << 2);
2660                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2661                 lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs);
2662                 lightrec_free_reg(reg_cache, rs);
2663                 return;
2664         }
2665
2666         unload_rd = OPT_EARLY_UNLOAD
2667                 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2668
2669         if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) {
2670                 /* The source register is not yet loaded - we can load its value
2671                  * from the register cache directly into the target register. */
2672                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2673
2674                 jit_ldxi_i(rd, LIGHTREC_REG_STATE,
2675                            offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2));
2676
2677                 lightrec_free_reg(reg_cache, rd);
2678         } else if (unload_rd) {
2679                 /* If the destination register will be unloaded right after the
2680                  * MOV meta-opcode, we don't actually need to write any host
2681                  * register - we can just store the source register directly to
2682                  * the register cache, at the offset corresponding to the
2683                  * destination register. */
2684                 lightrec_discard_reg_if_loaded(reg_cache, c.m.rd);
2685
2686                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2687
2688                 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
2689                            + (c.m.rd << 2), LIGHTREC_REG_STATE, rs);
2690
2691                 lightrec_free_reg(reg_cache, rs);
2692         } else {
2693                 if (c.m.rs)
2694                         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2695
2696                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2697
2698                 if (c.m.rs == 0) {
2699                         jit_movi(rd, 0);
2700                 } else {
2701                         jit_extr_i(rd, rs);
2702                         lightrec_free_reg(reg_cache, rs);
2703                 }
2704
2705                 lightrec_free_reg(reg_cache, rd);
2706         }
2707 }
2708
2709 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2710                                const struct block *block,
2711                                u16 offset)
2712 {
2713         struct regcache *reg_cache = state->reg_cache;
2714         union code c = block->opcode_list[offset].c;
2715         jit_state_t *_jit = block->_jit;
2716         u8 rs, rd;
2717
2718         _jit_name(block->_jit, __func__);
2719         jit_note(__FILE__, __LINE__);
2720
2721         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2722                         c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd);
2723
2724         if (c.m.op == OP_META_EXTC)
2725                 jit_extr_c(rd, rs);
2726         else
2727                 jit_extr_s(rd, rs);
2728
2729         lightrec_free_reg(reg_cache, rs);
2730         lightrec_free_reg(reg_cache, rd);
2731 }
2732
2733 static void rec_meta_MULT2(struct lightrec_cstate *state,
2734                            const struct block *block,
2735                            u16 offset)
2736 {
2737         struct regcache *reg_cache = state->reg_cache;
2738         union code c = block->opcode_list[offset].c;
2739         jit_state_t *_jit = block->_jit;
2740         u8 reg_lo = get_mult_div_lo(c);
2741         u8 reg_hi = get_mult_div_hi(c);
2742         u32 flags = block->opcode_list[offset].flags;
2743         bool is_signed = c.i.op == OP_META_MULT2;
2744         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2745         unsigned int i;
2746
2747         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2748                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2749                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2750         }
2751
2752         _jit_name(block->_jit, __func__);
2753         jit_note(__FILE__, __LINE__);
2754
2755         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2756
2757         /*
2758          * We must handle the case where one of the output registers is our rs
2759          * input register. Thanksfully, computing LO/HI can be done in any
2760          * order. Here, we make sure that the computation that overwrites the
2761          * input register is always performed last.
2762          */
2763         for (i = 0; i < 2; i++) {
2764                 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2765                         lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2766
2767                         if (c.r.op < 32)
2768                                 jit_lshi(lo, rs, c.r.op);
2769                         else
2770                                 jit_movi(lo, 0);
2771
2772                         lightrec_free_reg(reg_cache, lo);
2773                         continue;
2774                 }
2775
2776                 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2777                         hi = lightrec_alloc_reg_out(reg_cache, _jit,
2778                                                     reg_hi, hiflags);
2779
2780                         if (c.r.op >= 32) {
2781                                 jit_lshi(hi, rs, c.r.op - 32);
2782                         } else if (is_signed) {
2783                                 if (c.r.op)
2784                                         jit_rshi(hi, rs, 32 - c.r.op);
2785                                 else
2786                                         jit_rshi(hi, rs, 31);
2787                         } else {
2788                                 if (c.r.op)
2789                                         jit_rshi_u(hi, rs, 32 - c.r.op);
2790                                 else
2791                                         jit_movi(hi, 0);
2792                         }
2793
2794                         lightrec_free_reg(reg_cache, hi);
2795                 }
2796         }
2797
2798         lightrec_free_reg(reg_cache, rs);
2799
2800         _jit_name(block->_jit, __func__);
2801         jit_note(__FILE__, __LINE__);
2802 }
2803
2804 static void rec_meta_COM(struct lightrec_cstate *state,
2805                          const struct block *block, u16 offset)
2806 {
2807         struct regcache *reg_cache = state->reg_cache;
2808         union code c = block->opcode_list[offset].c;
2809         jit_state_t *_jit = block->_jit;
2810         u8 rd, rs, flags;
2811
2812         jit_note(__FILE__, __LINE__);
2813
2814         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2815                         c.m.rs, c.m.rd, 0, 0, &rs, &rd);
2816
2817         flags = lightrec_get_reg_in_flags(reg_cache, rs);
2818
2819         lightrec_set_reg_out_flags(reg_cache, rd,
2820                                    flags & REG_EXT);
2821
2822         jit_comr(rd, rs);
2823
2824         lightrec_free_reg(reg_cache, rs);
2825         lightrec_free_reg(reg_cache, rd);
2826 }
2827
2828 static void rec_meta_LWU(struct lightrec_cstate *state,
2829                          const struct block *block,
2830                          u16 offset)
2831 {
2832         jit_code_t code;
2833
2834         if (is_big_endian() && __WORDSIZE == 64)
2835                 code = jit_code_unldr_u;
2836         else
2837                 code = jit_code_unldr;
2838
2839         _jit_name(block->_jit, __func__);
2840         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2841 }
2842
2843 static void rec_meta_SWU(struct lightrec_cstate *state,
2844                          const struct block *block,
2845                          u16 offset)
2846 {
2847         _jit_name(block->_jit, __func__);
2848         rec_store(state, block, offset, jit_code_unstr, jit_code_bswapr_ui);
2849 }
2850
2851 static void unknown_opcode(struct lightrec_cstate *state,
2852                            const struct block *block, u16 offset)
2853 {
2854         rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP,
2855                        block->pc + (offset << 2));
2856 }
2857
2858 static const lightrec_rec_func_t rec_standard[64] = {
2859         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2860         [OP_SPECIAL]            = rec_SPECIAL,
2861         [OP_REGIMM]             = rec_REGIMM,
2862         [OP_J]                  = rec_J,
2863         [OP_JAL]                = rec_JAL,
2864         [OP_BEQ]                = rec_BEQ,
2865         [OP_BNE]                = rec_BNE,
2866         [OP_BLEZ]               = rec_BLEZ,
2867         [OP_BGTZ]               = rec_BGTZ,
2868         [OP_ADDI]               = rec_ADDI,
2869         [OP_ADDIU]              = rec_ADDIU,
2870         [OP_SLTI]               = rec_SLTI,
2871         [OP_SLTIU]              = rec_SLTIU,
2872         [OP_ANDI]               = rec_ANDI,
2873         [OP_ORI]                = rec_ORI,
2874         [OP_XORI]               = rec_XORI,
2875         [OP_LUI]                = rec_LUI,
2876         [OP_CP0]                = rec_CP0,
2877         [OP_CP2]                = rec_CP2,
2878         [OP_LB]                 = rec_LB,
2879         [OP_LH]                 = rec_LH,
2880         [OP_LWL]                = rec_LWL,
2881         [OP_LW]                 = rec_LW,
2882         [OP_LBU]                = rec_LBU,
2883         [OP_LHU]                = rec_LHU,
2884         [OP_LWR]                = rec_LWR,
2885         [OP_SB]                 = rec_SB,
2886         [OP_SH]                 = rec_SH,
2887         [OP_SWL]                = rec_SWL,
2888         [OP_SW]                 = rec_SW,
2889         [OP_SWR]                = rec_SWR,
2890         [OP_LWC2]               = rec_LW,
2891         [OP_SWC2]               = rec_SW,
2892
2893         [OP_META]               = rec_META,
2894         [OP_META_MULT2]         = rec_meta_MULT2,
2895         [OP_META_MULTU2]        = rec_meta_MULT2,
2896         [OP_META_LWU]           = rec_meta_LWU,
2897         [OP_META_SWU]           = rec_meta_SWU,
2898 };
2899
2900 static const lightrec_rec_func_t rec_special[64] = {
2901         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2902         [OP_SPECIAL_SLL]        = rec_special_SLL,
2903         [OP_SPECIAL_SRL]        = rec_special_SRL,
2904         [OP_SPECIAL_SRA]        = rec_special_SRA,
2905         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2906         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2907         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2908         [OP_SPECIAL_JR]         = rec_special_JR,
2909         [OP_SPECIAL_JALR]       = rec_special_JALR,
2910         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2911         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2912         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2913         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2914         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2915         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2916         [OP_SPECIAL_MULT]       = rec_special_MULT,
2917         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2918         [OP_SPECIAL_DIV]        = rec_special_DIV,
2919         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2920         [OP_SPECIAL_ADD]        = rec_special_ADD,
2921         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2922         [OP_SPECIAL_SUB]        = rec_special_SUB,
2923         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2924         [OP_SPECIAL_AND]        = rec_special_AND,
2925         [OP_SPECIAL_OR]         = rec_special_OR,
2926         [OP_SPECIAL_XOR]        = rec_special_XOR,
2927         [OP_SPECIAL_NOR]        = rec_special_NOR,
2928         [OP_SPECIAL_SLT]        = rec_special_SLT,
2929         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2930 };
2931
2932 static const lightrec_rec_func_t rec_regimm[64] = {
2933         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2934         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2935         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2936         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2937         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2938 };
2939
2940 static const lightrec_rec_func_t rec_cp0[64] = {
2941         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2942         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2943         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2944         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2945         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2946         [OP_CP0_RFE]            = rec_cp0_RFE,
2947 };
2948
2949 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2950         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2951         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2952         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2953         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2954         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2955 };
2956
2957 static const lightrec_rec_func_t rec_meta[64] = {
2958         SET_DEFAULT_ELM(rec_meta, unknown_opcode),
2959         [OP_META_MOV]           = rec_meta_MOV,
2960         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2961         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2962         [OP_META_COM]           = rec_meta_COM,
2963 };
2964
2965 static void rec_SPECIAL(struct lightrec_cstate *state,
2966                         const struct block *block, u16 offset)
2967 {
2968         union code c = block->opcode_list[offset].c;
2969         lightrec_rec_func_t f = rec_special[c.r.op];
2970
2971         if (!HAS_DEFAULT_ELM && unlikely(!f))
2972                 unknown_opcode(state, block, offset);
2973         else
2974                 (*f)(state, block, offset);
2975 }
2976
2977 static void rec_REGIMM(struct lightrec_cstate *state,
2978                        const struct block *block, u16 offset)
2979 {
2980         union code c = block->opcode_list[offset].c;
2981         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2982
2983         if (!HAS_DEFAULT_ELM && unlikely(!f))
2984                 unknown_opcode(state, block, offset);
2985         else
2986                 (*f)(state, block, offset);
2987 }
2988
2989 static void rec_CP0(struct lightrec_cstate *state,
2990                     const struct block *block, u16 offset)
2991 {
2992         union code c = block->opcode_list[offset].c;
2993         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2994
2995         if (!HAS_DEFAULT_ELM && unlikely(!f))
2996                 rec_CP(state, block, offset);
2997         else
2998                 (*f)(state, block, offset);
2999 }
3000
3001 static void rec_CP2(struct lightrec_cstate *state,
3002                     const struct block *block, u16 offset)
3003 {
3004         union code c = block->opcode_list[offset].c;
3005
3006         if (c.r.op == OP_CP2_BASIC) {
3007                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
3008
3009                 if (HAS_DEFAULT_ELM || likely(f)) {
3010                         (*f)(state, block, offset);
3011                         return;
3012                 }
3013         }
3014
3015         rec_CP(state, block, offset);
3016 }
3017
3018 static void rec_META(struct lightrec_cstate *state,
3019                      const struct block *block, u16 offset)
3020 {
3021         union code c = block->opcode_list[offset].c;
3022         lightrec_rec_func_t f = rec_meta[c.m.op];
3023
3024         if (!HAS_DEFAULT_ELM && unlikely(!f))
3025                 unknown_opcode(state, block, offset);
3026         else
3027                 (*f)(state, block, offset);
3028 }
3029
3030 void lightrec_rec_opcode(struct lightrec_cstate *state,
3031                          const struct block *block, u16 offset)
3032 {
3033         struct regcache *reg_cache = state->reg_cache;
3034         struct lightrec_branch_target *target;
3035         const struct opcode *op = &block->opcode_list[offset];
3036         jit_state_t *_jit = block->_jit;
3037         lightrec_rec_func_t f;
3038         u16 unload_offset;
3039
3040         if (op_flag_sync(op->flags)) {
3041                 if (state->cycles)
3042                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
3043                 state->cycles = 0;
3044
3045                 lightrec_storeback_regs(reg_cache, _jit);
3046                 lightrec_regcache_reset(reg_cache);
3047
3048                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
3049                 target = &state->targets[state->nb_targets++];
3050                 target->offset = offset;
3051                 target->label = jit_indirect();
3052         }
3053
3054         if (likely(op->opcode)) {
3055                 f = rec_standard[op->i.op];
3056
3057                 if (!HAS_DEFAULT_ELM && unlikely(!f))
3058                         unknown_opcode(state, block, offset);
3059                 else
3060                         (*f)(state, block, offset);
3061         }
3062
3063         if (OPT_EARLY_UNLOAD) {
3064                 unload_offset = offset +
3065                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
3066
3067                 lightrec_do_early_unload(state, block, unload_offset);
3068         }
3069
3070         state->no_load_delay = false;
3071 }