Update lightrec 20220911 (#687)
[pcsx_rearmed.git] / deps / lightrec / emitter.c
1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 /*
3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
4  */
5
6 #include "blockcache.h"
7 #include "debug.h"
8 #include "disassembler.h"
9 #include "emitter.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
12 #include "regcache.h"
13
14 #include <stdbool.h>
15 #include <stddef.h>
16
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
18
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
24
25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
26 {
27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
28                 block->opcode_list[offset].c.opcode,
29                 block->pc + (offset << 2));
30 }
31
32 static void
33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
34 {
35         /* Prevent jit_jmpi() from using our cycles register as a temporary */
36         jit_live(LIGHTREC_REG_CYCLE);
37
38         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
39 }
40
41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
42                                        const struct block *block, u16 offset,
43                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
44                                        u32 link, bool update_cycles)
45 {
46         struct regcache *reg_cache = state->reg_cache;
47         jit_state_t *_jit = block->_jit;
48         const struct opcode *op = &block->opcode_list[offset],
49                             *next = &block->opcode_list[offset + 1];
50         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
51
52         jit_note(__FILE__, __LINE__);
53
54         if (link) {
55                 /* Update the $ra register */
56                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
57                 jit_movi(link_reg, link);
58                 lightrec_free_reg(reg_cache, link_reg);
59         }
60
61         if (reg_new_pc < 0) {
62                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
63                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
64
65                 jit_movi(reg_new_pc, imm);
66         }
67
68         if (has_delay_slot(op->c) &&
69             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
70                 cycles += lightrec_cycles_of_opcode(next->c);
71
72                 /* Recompile the delay slot */
73                 if (next->c.opcode)
74                         lightrec_rec_opcode(state, block, offset + 1);
75         }
76
77         /* Clean the remaining registers */
78         lightrec_clean_regs(reg_cache, _jit);
79
80         jit_movr(JIT_V0, reg_new_pc);
81
82         if (cycles && update_cycles) {
83                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
84                 pr_debug("EOB: %u cycles\n", cycles);
85         }
86
87         lightrec_jump_to_eob(state, _jit);
88 }
89
90 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
91                        u16 offset, bool after_op)
92 {
93         struct regcache *reg_cache = state->reg_cache;
94         jit_state_t *_jit = block->_jit;
95         union code c = block->opcode_list[offset].c;
96         u32 cycles = state->cycles;
97
98         if (after_op)
99                 cycles += lightrec_cycles_of_opcode(c);
100
101         lightrec_clean_regs(reg_cache, _jit);
102
103         jit_movi(JIT_V0, block->pc + (offset << 2));
104         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
105
106         lightrec_jump_to_eob(state, _jit);
107 }
108
109 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
110 {
111         struct regcache *reg_cache = state->reg_cache;
112         jit_state_t *_jit = block->_jit;
113         const struct opcode *op = &block->opcode_list[offset];
114         u8 rs;
115
116         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
117         lightrec_lock_reg(reg_cache, _jit, rs);
118
119         return rs;
120 }
121
122 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
123 {
124         u8 rs = get_jr_jalr_reg(state, block, offset);
125
126         _jit_name(block->_jit, __func__);
127         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
128 }
129
130 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
131 {
132         u8 rs = get_jr_jalr_reg(state, block, offset);
133         union code c = block->opcode_list[offset].c;
134
135         _jit_name(block->_jit, __func__);
136         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
137                                    get_branch_pc(block, offset, 2), true);
138 }
139
140 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
141 {
142         union code c = block->opcode_list[offset].c;
143
144         _jit_name(block->_jit, __func__);
145         lightrec_emit_end_of_block(state, block, offset, -1,
146                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
147                                    31, 0, true);
148 }
149
150 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
151 {
152         union code c = block->opcode_list[offset].c;
153
154         _jit_name(block->_jit, __func__);
155         lightrec_emit_end_of_block(state, block, offset, -1,
156                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
157                                    31, get_branch_pc(block, offset, 2), true);
158 }
159
160 static void lightrec_do_early_unload(struct lightrec_cstate *state,
161                                      const struct block *block, u16 offset)
162 {
163         struct regcache *reg_cache = state->reg_cache;
164         const struct opcode *op = &block->opcode_list[offset];
165         jit_state_t *_jit = block->_jit;
166         unsigned int i;
167         u8 reg;
168         struct {
169                 u8 reg, op;
170         } reg_ops[3] = {
171                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
172                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
173                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
174         };
175
176         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
177                 reg = reg_ops[i].reg;
178
179                 switch (reg_ops[i].op) {
180                 case LIGHTREC_REG_UNLOAD:
181                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
182                         break;
183
184                 case LIGHTREC_REG_DISCARD:
185                         lightrec_discard_reg_if_loaded(reg_cache, reg);
186                         break;
187
188                 case LIGHTREC_REG_CLEAN:
189                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
190                         break;
191                 default:
192                         break;
193                 };
194         }
195 }
196
197 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
198                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
199 {
200         struct regcache *reg_cache = state->reg_cache;
201         struct native_register *regs_backup;
202         jit_state_t *_jit = block->_jit;
203         struct lightrec_branch *branch;
204         const struct opcode *op = &block->opcode_list[offset],
205                             *next = &block->opcode_list[offset + 1];
206         jit_node_t *addr;
207         u8 link_reg, rs, rt;
208         bool is_forward = (s16)op->i.imm >= -1;
209         int op_cycles = lightrec_cycles_of_opcode(op->c);
210         u32 target_offset, cycles = state->cycles + op_cycles;
211         bool no_indirection = false;
212         u32 next_pc;
213
214         jit_note(__FILE__, __LINE__);
215
216         if (!op_flag_no_ds(op->flags))
217                 cycles += lightrec_cycles_of_opcode(next->c);
218
219         state->cycles = -op_cycles;
220
221         if (!unconditional) {
222                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
223                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
224                                                     _jit, op->i.rt, REG_EXT);
225
226                 /* Unload dead registers before evaluating the branch */
227                 if (OPT_EARLY_UNLOAD)
228                         lightrec_do_early_unload(state, block, offset);
229
230                 if (op_flag_local_branch(op->flags) &&
231                     (op_flag_no_ds(op->flags) || !next->opcode) &&
232                     is_forward && !lightrec_has_dirty_regs(reg_cache))
233                         no_indirection = true;
234
235                 if (no_indirection)
236                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
237         }
238
239         if (cycles)
240                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
241
242         if (!unconditional) {
243                 /* Generate the branch opcode */
244                 if (!no_indirection)
245                         addr = jit_new_node_pww(code, NULL, rs, rt);
246
247                 lightrec_free_regs(reg_cache);
248                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
249         }
250
251         if (op_flag_local_branch(op->flags)) {
252                 /* Recompile the delay slot */
253                 if (next && next->opcode && !op_flag_no_ds(op->flags))
254                         lightrec_rec_opcode(state, block, offset + 1);
255
256                 if (link) {
257                         /* Update the $ra register */
258                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
259                         jit_movi(link_reg, link);
260                         lightrec_free_reg(reg_cache, link_reg);
261                 }
262
263                 /* Clean remaining registers */
264                 lightrec_clean_regs(reg_cache, _jit);
265
266                 target_offset = offset + 1 + (s16)op->i.imm
267                         - !!op_flag_no_ds(op->flags);
268                 pr_debug("Adding local branch to offset 0x%x\n",
269                          target_offset << 2);
270                 branch = &state->local_branches[
271                         state->nb_local_branches++];
272
273                 branch->target = target_offset;
274
275                 if (no_indirection)
276                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
277                 else if (is_forward)
278                         branch->branch = jit_b();
279                 else
280                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
281         }
282
283         if (!op_flag_local_branch(op->flags) || !is_forward) {
284                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
285                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
286                                            31, link, false);
287         }
288
289         if (!unconditional) {
290                 if (!no_indirection)
291                         jit_patch(addr);
292
293                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
294
295                 if (bz && link) {
296                         /* Update the $ra register */
297                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
298                                                           31, REG_EXT);
299                         jit_movi(link_reg, (s32)link);
300                         lightrec_free_reg(reg_cache, link_reg);
301                 }
302
303                 if (!op_flag_no_ds(op->flags) && next->opcode)
304                         lightrec_rec_opcode(state, block, offset + 1);
305         }
306 }
307
308 static void rec_BNE(struct lightrec_cstate *state,
309                     const struct block *block, u16 offset)
310 {
311         union code c = block->opcode_list[offset].c;
312
313         _jit_name(block->_jit, __func__);
314
315         if (c.i.rt == 0)
316                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
317         else
318                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
319 }
320
321 static void rec_BEQ(struct lightrec_cstate *state,
322                     const struct block *block, u16 offset)
323 {
324         union code c = block->opcode_list[offset].c;
325
326         _jit_name(block->_jit, __func__);
327
328         if (c.i.rt == 0)
329                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
330         else
331                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
332 }
333
334 static void rec_BLEZ(struct lightrec_cstate *state,
335                      const struct block *block, u16 offset)
336 {
337         union code c = block->opcode_list[offset].c;
338
339         _jit_name(block->_jit, __func__);
340         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
341 }
342
343 static void rec_BGTZ(struct lightrec_cstate *state,
344                      const struct block *block, u16 offset)
345 {
346         _jit_name(block->_jit, __func__);
347         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
348 }
349
350 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
351                             const struct block *block, u16 offset)
352 {
353         _jit_name(block->_jit, __func__);
354         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
355 }
356
357 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
358                               const struct block *block, u16 offset)
359 {
360         _jit_name(block->_jit, __func__);
361         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
362               get_branch_pc(block, offset, 2), false, true);
363 }
364
365 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
366                             const struct block *block, u16 offset)
367 {
368         union code c = block->opcode_list[offset].c;
369
370         _jit_name(block->_jit, __func__);
371         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
372 }
373
374 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
375                               const struct block *block, u16 offset)
376 {
377         const struct opcode *op = &block->opcode_list[offset];
378         _jit_name(block->_jit, __func__);
379         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
380               get_branch_pc(block, offset, 2),
381               !op->i.rs, true);
382 }
383
384 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
385                         u16 offset, jit_code_t code, bool slti)
386 {
387         struct regcache *reg_cache = state->reg_cache;
388         union code c = block->opcode_list[offset].c;
389         jit_state_t *_jit = block->_jit;
390         u8 rs, rt, out_flags = REG_EXT;
391
392         if (slti)
393                 out_flags |= REG_ZEXT;
394
395         jit_note(__FILE__, __LINE__);
396         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
397         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
398
399         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
400
401         lightrec_free_reg(reg_cache, rs);
402         lightrec_free_reg(reg_cache, rt);
403 }
404
405 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
406                             u16 offset, jit_code_t code, bool out_ext)
407 {
408         struct regcache *reg_cache = state->reg_cache;
409         union code c = block->opcode_list[offset].c;
410         jit_state_t *_jit = block->_jit;
411         u8 rd, rt, rs;
412
413         jit_note(__FILE__, __LINE__);
414         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
415         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
416         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
417                                     out_ext ? REG_EXT | REG_ZEXT : 0);
418
419         jit_new_node_www(code, rd, rs, rt);
420
421         lightrec_free_reg(reg_cache, rs);
422         lightrec_free_reg(reg_cache, rt);
423         lightrec_free_reg(reg_cache, rd);
424 }
425
426 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
427                            u16 offset, jit_code_t code)
428 {
429         struct regcache *reg_cache = state->reg_cache;
430         union code c = block->opcode_list[offset].c;
431         jit_state_t *_jit = block->_jit;
432         u8 rd, rt, rs, temp, flags = 0;
433
434         jit_note(__FILE__, __LINE__);
435         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
436
437         if (code == jit_code_rshr)
438                 flags = REG_EXT;
439         else if (code == jit_code_rshr_u)
440                 flags = REG_ZEXT;
441
442         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
443         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
444
445         if (rs != rd && rt != rd) {
446                 jit_andi(rd, rs, 0x1f);
447                 jit_new_node_www(code, rd, rt, rd);
448         } else {
449                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
450                 jit_andi(temp, rs, 0x1f);
451                 jit_new_node_www(code, rd, rt, temp);
452                 lightrec_free_reg(reg_cache, temp);
453         }
454
455         lightrec_free_reg(reg_cache, rs);
456         lightrec_free_reg(reg_cache, rt);
457         lightrec_free_reg(reg_cache, rd);
458 }
459
460 static void rec_movi(struct lightrec_cstate *state,
461                      const struct block *block, u16 offset)
462 {
463         struct regcache *reg_cache = state->reg_cache;
464         union code c = block->opcode_list[offset].c;
465         jit_state_t *_jit = block->_jit;
466         u16 flags = REG_EXT;
467         u8 rt;
468
469         if (!(c.i.imm & 0x8000))
470                 flags |= REG_ZEXT;
471
472         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
473
474         jit_movi(rt, (s32)(s16) c.i.imm);
475
476         lightrec_free_reg(reg_cache, rt);
477 }
478
479 static void rec_ADDIU(struct lightrec_cstate *state,
480                       const struct block *block, u16 offset)
481 {
482         _jit_name(block->_jit, __func__);
483
484         if (block->opcode_list[offset].c.i.rs)
485                 rec_alu_imm(state, block, offset, jit_code_addi, false);
486         else
487                 rec_movi(state, block, offset);
488 }
489
490 static void rec_ADDI(struct lightrec_cstate *state,
491                      const struct block *block, u16 offset)
492 {
493         /* TODO: Handle the exception? */
494         _jit_name(block->_jit, __func__);
495         rec_ADDIU(state, block, offset);
496 }
497
498 static void rec_SLTIU(struct lightrec_cstate *state,
499                       const struct block *block, u16 offset)
500 {
501         _jit_name(block->_jit, __func__);
502         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
503 }
504
505 static void rec_SLTI(struct lightrec_cstate *state,
506                      const struct block *block, u16 offset)
507 {
508         _jit_name(block->_jit, __func__);
509         rec_alu_imm(state, block, offset, jit_code_lti, true);
510 }
511
512 static void rec_ANDI(struct lightrec_cstate *state,
513                      const struct block *block, u16 offset)
514 {
515         struct regcache *reg_cache = state->reg_cache;
516         union code c = block->opcode_list[offset].c;
517         jit_state_t *_jit = block->_jit;
518         u8 rs, rt;
519
520         _jit_name(block->_jit, __func__);
521         jit_note(__FILE__, __LINE__);
522         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
523         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
524                                     REG_EXT | REG_ZEXT);
525
526         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
527          * casts to uint8_t / uint16_t. */
528         if (c.i.imm == 0xff)
529                 jit_extr_uc(rt, rs);
530         else if (c.i.imm == 0xffff)
531                 jit_extr_us(rt, rs);
532         else
533                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
534
535         lightrec_free_reg(reg_cache, rs);
536         lightrec_free_reg(reg_cache, rt);
537 }
538
539 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
540                            u16 offset, jit_code_t code)
541 {
542         struct regcache *reg_cache = state->reg_cache;
543         union code c = block->opcode_list[offset].c;
544         jit_state_t *_jit = block->_jit;
545         u8 rs, rt, flags;
546
547         jit_note(__FILE__, __LINE__);
548         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
549         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
550
551         flags = lightrec_get_reg_in_flags(reg_cache, rs);
552         lightrec_set_reg_out_flags(reg_cache, rt, flags);
553
554         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
555
556         lightrec_free_reg(reg_cache, rs);
557         lightrec_free_reg(reg_cache, rt);
558 }
559
560
561 static void rec_ORI(struct lightrec_cstate *state,
562                     const struct block *block, u16 offset)
563 {
564         _jit_name(block->_jit, __func__);
565         rec_alu_or_xor(state, block, offset, jit_code_ori);
566 }
567
568 static void rec_XORI(struct lightrec_cstate *state,
569                      const struct block *block, u16 offset)
570 {
571         _jit_name(block->_jit, __func__);
572         rec_alu_or_xor(state, block, offset, jit_code_xori);
573 }
574
575 static void rec_LUI(struct lightrec_cstate *state,
576                     const struct block *block, u16 offset)
577 {
578         struct regcache *reg_cache = state->reg_cache;
579         union code c = block->opcode_list[offset].c;
580         jit_state_t *_jit = block->_jit;
581         u8 rt, flags = REG_EXT;
582
583         jit_name(__func__);
584         jit_note(__FILE__, __LINE__);
585
586         if (!(c.i.imm & BIT(15)))
587                 flags |= REG_ZEXT;
588
589         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
590
591         jit_movi(rt, (s32)(c.i.imm << 16));
592
593         lightrec_free_reg(reg_cache, rt);
594 }
595
596 static void rec_special_ADDU(struct lightrec_cstate *state,
597                              const struct block *block, u16 offset)
598 {
599         _jit_name(block->_jit, __func__);
600         rec_alu_special(state, block, offset, jit_code_addr, false);
601 }
602
603 static void rec_special_ADD(struct lightrec_cstate *state,
604                             const struct block *block, u16 offset)
605 {
606         /* TODO: Handle the exception? */
607         _jit_name(block->_jit, __func__);
608         rec_alu_special(state, block, offset, jit_code_addr, false);
609 }
610
611 static void rec_special_SUBU(struct lightrec_cstate *state,
612                              const struct block *block, u16 offset)
613 {
614         _jit_name(block->_jit, __func__);
615         rec_alu_special(state, block, offset, jit_code_subr, false);
616 }
617
618 static void rec_special_SUB(struct lightrec_cstate *state,
619                             const struct block *block, u16 offset)
620 {
621         /* TODO: Handle the exception? */
622         _jit_name(block->_jit, __func__);
623         rec_alu_special(state, block, offset, jit_code_subr, false);
624 }
625
626 static void rec_special_AND(struct lightrec_cstate *state,
627                             const struct block *block, u16 offset)
628 {
629         struct regcache *reg_cache = state->reg_cache;
630         union code c = block->opcode_list[offset].c;
631         jit_state_t *_jit = block->_jit;
632         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
633
634         _jit_name(block->_jit, __func__);
635         jit_note(__FILE__, __LINE__);
636         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
637         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
638         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
639
640         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
641         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
642
643         /* Z(rd) = Z(rs) | Z(rt) */
644         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
645
646         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
647         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
648             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
649             (REG_EXT & flags_rs & flags_rt))
650                 flags_rd |= REG_EXT;
651
652         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
653
654         jit_andr(rd, rs, rt);
655
656         lightrec_free_reg(reg_cache, rs);
657         lightrec_free_reg(reg_cache, rt);
658         lightrec_free_reg(reg_cache, rd);
659 }
660
661 static void rec_special_or_nor(struct lightrec_cstate *state,
662                                const struct block *block, u16 offset, bool nor)
663 {
664         struct regcache *reg_cache = state->reg_cache;
665         union code c = block->opcode_list[offset].c;
666         jit_state_t *_jit = block->_jit;
667         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
668
669         jit_note(__FILE__, __LINE__);
670         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
671         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
672         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
673
674         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
675         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
676
677         /* or: Z(rd) = Z(rs) & Z(rt)
678          * nor: Z(rd) = 0 */
679         if (!nor)
680                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
681
682         /* E(rd) = E(rs) & E(rt) */
683         if (REG_EXT & flags_rs & flags_rt)
684                 flags_rd |= REG_EXT;
685
686         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
687
688         jit_orr(rd, rs, rt);
689
690         if (nor)
691                 jit_comr(rd, rd);
692
693         lightrec_free_reg(reg_cache, rs);
694         lightrec_free_reg(reg_cache, rt);
695         lightrec_free_reg(reg_cache, rd);
696 }
697
698 static void rec_special_OR(struct lightrec_cstate *state,
699                            const struct block *block, u16 offset)
700 {
701         _jit_name(block->_jit, __func__);
702         rec_special_or_nor(state, block, offset, false);
703 }
704
705 static void rec_special_NOR(struct lightrec_cstate *state,
706                             const struct block *block, u16 offset)
707 {
708         _jit_name(block->_jit, __func__);
709         rec_special_or_nor(state, block, offset, true);
710 }
711
712 static void rec_special_XOR(struct lightrec_cstate *state,
713                             const struct block *block, u16 offset)
714 {
715         struct regcache *reg_cache = state->reg_cache;
716         union code c = block->opcode_list[offset].c;
717         jit_state_t *_jit = block->_jit;
718         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
719
720         _jit_name(block->_jit, __func__);
721
722         jit_note(__FILE__, __LINE__);
723         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
724         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
725         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
726
727         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
728         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
729
730         /* Z(rd) = Z(rs) & Z(rt) */
731         flags_rd = REG_ZEXT & flags_rs & flags_rt;
732
733         /* E(rd) = E(rs) & E(rt) */
734         flags_rd |= REG_EXT & flags_rs & flags_rt;
735
736         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
737
738         jit_xorr(rd, rs, rt);
739
740         lightrec_free_reg(reg_cache, rs);
741         lightrec_free_reg(reg_cache, rt);
742         lightrec_free_reg(reg_cache, rd);
743 }
744
745 static void rec_special_SLTU(struct lightrec_cstate *state,
746                              const struct block *block, u16 offset)
747 {
748         _jit_name(block->_jit, __func__);
749         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
750 }
751
752 static void rec_special_SLT(struct lightrec_cstate *state,
753                             const struct block *block, u16 offset)
754 {
755         _jit_name(block->_jit, __func__);
756         rec_alu_special(state, block, offset, jit_code_ltr, true);
757 }
758
759 static void rec_special_SLLV(struct lightrec_cstate *state,
760                              const struct block *block, u16 offset)
761 {
762         _jit_name(block->_jit, __func__);
763         rec_alu_shiftv(state, block, offset, jit_code_lshr);
764 }
765
766 static void rec_special_SRLV(struct lightrec_cstate *state,
767                              const struct block *block, u16 offset)
768 {
769         _jit_name(block->_jit, __func__);
770         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
771 }
772
773 static void rec_special_SRAV(struct lightrec_cstate *state,
774                              const struct block *block, u16 offset)
775 {
776         _jit_name(block->_jit, __func__);
777         rec_alu_shiftv(state, block, offset, jit_code_rshr);
778 }
779
780 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
781                           u16 offset, jit_code_t code)
782 {
783         struct regcache *reg_cache = state->reg_cache;
784         union code c = block->opcode_list[offset].c;
785         jit_state_t *_jit = block->_jit;
786         u8 rd, rt, flags = 0;
787
788         jit_note(__FILE__, __LINE__);
789
790         if (code == jit_code_rshi)
791                 flags = REG_EXT;
792         else if (code == jit_code_rshi_u)
793                 flags = REG_ZEXT;
794
795         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
796
797         /* Input reg is zero-extended, if we SRL at least by one bit, we know
798          * the output reg will be both zero-extended and sign-extended. */
799         if (code == jit_code_rshi_u && c.r.imm)
800                 flags |= REG_EXT;
801         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
802
803         jit_new_node_www(code, rd, rt, c.r.imm);
804
805         lightrec_free_reg(reg_cache, rt);
806         lightrec_free_reg(reg_cache, rd);
807 }
808
809 static void rec_special_SLL(struct lightrec_cstate *state,
810                             const struct block *block, u16 offset)
811 {
812         _jit_name(block->_jit, __func__);
813         rec_alu_shift(state, block, offset, jit_code_lshi);
814 }
815
816 static void rec_special_SRL(struct lightrec_cstate *state,
817                             const struct block *block, u16 offset)
818 {
819         _jit_name(block->_jit, __func__);
820         rec_alu_shift(state, block, offset, jit_code_rshi_u);
821 }
822
823 static void rec_special_SRA(struct lightrec_cstate *state,
824                             const struct block *block, u16 offset)
825 {
826         _jit_name(block->_jit, __func__);
827         rec_alu_shift(state, block, offset, jit_code_rshi);
828 }
829
830 static void rec_alu_mult(struct lightrec_cstate *state,
831                          const struct block *block, u16 offset, bool is_signed)
832 {
833         struct regcache *reg_cache = state->reg_cache;
834         union code c = block->opcode_list[offset].c;
835         u32 flags = block->opcode_list[offset].flags;
836         u8 reg_lo = get_mult_div_lo(c);
837         u8 reg_hi = get_mult_div_hi(c);
838         jit_state_t *_jit = block->_jit;
839         u8 lo, hi, rs, rt, rflags = 0;
840
841         jit_note(__FILE__, __LINE__);
842
843         if (is_signed)
844                 rflags = REG_EXT;
845         else
846                 rflags = REG_ZEXT;
847
848         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
849         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
850
851         if (!op_flag_no_lo(flags))
852                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
853         else if (__WORDSIZE == 32)
854                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
855
856         if (!op_flag_no_hi(flags))
857                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
858
859         if (__WORDSIZE == 32) {
860                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
861                  * operation if the MULT was detected a 32-bit only. */
862                 if (!op_flag_no_hi(flags)) {
863                         if (is_signed)
864                                 jit_qmulr(lo, hi, rs, rt);
865                         else
866                                 jit_qmulr_u(lo, hi, rs, rt);
867                 } else {
868                         jit_mulr(lo, rs, rt);
869                 }
870         } else {
871                 /* On 64-bit systems, do a 64*64->64 bit operation. */
872                 if (op_flag_no_lo(flags)) {
873                         jit_mulr(hi, rs, rt);
874                         jit_rshi(hi, hi, 32);
875                 } else {
876                         jit_mulr(lo, rs, rt);
877
878                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
879                         if (!op_flag_no_hi(flags))
880                                 jit_rshi(hi, lo, 32);
881                 }
882         }
883
884         lightrec_free_reg(reg_cache, rs);
885         lightrec_free_reg(reg_cache, rt);
886         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
887                 lightrec_free_reg(reg_cache, lo);
888         if (!op_flag_no_hi(flags))
889                 lightrec_free_reg(reg_cache, hi);
890 }
891
892 static void rec_alu_div(struct lightrec_cstate *state,
893                         const struct block *block, u16 offset, bool is_signed)
894 {
895         struct regcache *reg_cache = state->reg_cache;
896         union code c = block->opcode_list[offset].c;
897         u32 flags = block->opcode_list[offset].flags;
898         bool no_check = op_flag_no_div_check(flags);
899         u8 reg_lo = get_mult_div_lo(c);
900         u8 reg_hi = get_mult_div_hi(c);
901         jit_state_t *_jit = block->_jit;
902         jit_node_t *branch, *to_end;
903         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
904
905         jit_note(__FILE__, __LINE__);
906
907         if (is_signed)
908                 rflags = REG_EXT;
909         else
910                 rflags = REG_ZEXT;
911
912         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
913         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
914
915         if (!op_flag_no_lo(flags))
916                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
917
918         if (!op_flag_no_hi(flags))
919                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
920
921         /* Jump to special handler if dividing by zero  */
922         if (!no_check)
923                 branch = jit_beqi(rt, 0);
924
925         if (op_flag_no_lo(flags)) {
926                 if (is_signed)
927                         jit_remr(hi, rs, rt);
928                 else
929                         jit_remr_u(hi, rs, rt);
930         } else if (op_flag_no_hi(flags)) {
931                 if (is_signed)
932                         jit_divr(lo, rs, rt);
933                 else
934                         jit_divr_u(lo, rs, rt);
935         } else {
936                 if (is_signed)
937                         jit_qdivr(lo, hi, rs, rt);
938                 else
939                         jit_qdivr_u(lo, hi, rs, rt);
940         }
941
942         if (!no_check) {
943                 /* Jump above the div-by-zero handler */
944                 to_end = jit_b();
945
946                 jit_patch(branch);
947
948                 if (!op_flag_no_lo(flags)) {
949                         if (is_signed) {
950                                 jit_lti(lo, rs, 0);
951                                 jit_lshi(lo, lo, 1);
952                                 jit_subi(lo, lo, 1);
953                         } else {
954                                 jit_movi(lo, 0xffffffff);
955                         }
956                 }
957
958                 if (!op_flag_no_hi(flags))
959                         jit_movr(hi, rs);
960
961                 jit_patch(to_end);
962         }
963
964         lightrec_free_reg(reg_cache, rs);
965         lightrec_free_reg(reg_cache, rt);
966
967         if (!op_flag_no_lo(flags))
968                 lightrec_free_reg(reg_cache, lo);
969
970         if (!op_flag_no_hi(flags))
971                 lightrec_free_reg(reg_cache, hi);
972 }
973
974 static void rec_special_MULT(struct lightrec_cstate *state,
975                              const struct block *block, u16 offset)
976 {
977         _jit_name(block->_jit, __func__);
978         rec_alu_mult(state, block, offset, true);
979 }
980
981 static void rec_special_MULTU(struct lightrec_cstate *state,
982                               const struct block *block, u16 offset)
983 {
984         _jit_name(block->_jit, __func__);
985         rec_alu_mult(state, block, offset, false);
986 }
987
988 static void rec_special_DIV(struct lightrec_cstate *state,
989                             const struct block *block, u16 offset)
990 {
991         _jit_name(block->_jit, __func__);
992         rec_alu_div(state, block, offset, true);
993 }
994
995 static void rec_special_DIVU(struct lightrec_cstate *state,
996                              const struct block *block, u16 offset)
997 {
998         _jit_name(block->_jit, __func__);
999         rec_alu_div(state, block, offset, false);
1000 }
1001
1002 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1003                              const struct block *block, u8 dst, u8 src)
1004 {
1005         struct regcache *reg_cache = state->reg_cache;
1006         jit_state_t *_jit = block->_jit;
1007
1008         jit_note(__FILE__, __LINE__);
1009         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1010         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1011
1012         jit_extr_i(dst, src);
1013
1014         lightrec_free_reg(reg_cache, src);
1015         lightrec_free_reg(reg_cache, dst);
1016 }
1017
1018 static void rec_special_MFHI(struct lightrec_cstate *state,
1019                              const struct block *block, u16 offset)
1020 {
1021         union code c = block->opcode_list[offset].c;
1022
1023         _jit_name(block->_jit, __func__);
1024         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1025 }
1026
1027 static void rec_special_MTHI(struct lightrec_cstate *state,
1028                              const struct block *block, u16 offset)
1029 {
1030         union code c = block->opcode_list[offset].c;
1031
1032         _jit_name(block->_jit, __func__);
1033         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1034 }
1035
1036 static void rec_special_MFLO(struct lightrec_cstate *state,
1037                              const struct block *block, u16 offset)
1038 {
1039         union code c = block->opcode_list[offset].c;
1040
1041         _jit_name(block->_jit, __func__);
1042         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1043 }
1044
1045 static void rec_special_MTLO(struct lightrec_cstate *state,
1046                              const struct block *block, u16 offset)
1047 {
1048         union code c = block->opcode_list[offset].c;
1049
1050         _jit_name(block->_jit, __func__);
1051         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1052 }
1053
1054 static void call_to_c_wrapper(struct lightrec_cstate *state,
1055                               const struct block *block, u32 arg,
1056                               enum c_wrappers wrapper)
1057 {
1058         struct regcache *reg_cache = state->reg_cache;
1059         jit_state_t *_jit = block->_jit;
1060         s8 tmp, tmp2;
1061
1062         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1063         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1064
1065         tmp = lightrec_get_reg_with_value(reg_cache,
1066                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1067         if (tmp < 0) {
1068                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1069                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1070                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1071
1072                 lightrec_temp_set_value(reg_cache, tmp,
1073                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1074         }
1075
1076         lightrec_free_reg(reg_cache, tmp2);
1077
1078 #ifdef __mips__
1079         /* On MIPS, register t9 is always used as the target register for JALR.
1080          * Therefore if it does not contain the target address we must
1081          * invalidate it. */
1082         if (tmp != _T9)
1083                 lightrec_unload_reg(reg_cache, _jit, _T9);
1084 #endif
1085
1086         jit_prepare();
1087         jit_pushargi(arg);
1088
1089         lightrec_regcache_mark_live(reg_cache, _jit);
1090         jit_callr(tmp);
1091
1092         lightrec_free_reg(reg_cache, tmp);
1093         lightrec_regcache_mark_live(reg_cache, _jit);
1094 }
1095
1096 static void rec_io(struct lightrec_cstate *state,
1097                    const struct block *block, u16 offset,
1098                    bool load_rt, bool read_rt)
1099 {
1100         struct regcache *reg_cache = state->reg_cache;
1101         jit_state_t *_jit = block->_jit;
1102         union code c = block->opcode_list[offset].c;
1103         u32 flags = block->opcode_list[offset].flags;
1104         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1105         u32 lut_entry;
1106
1107         jit_note(__FILE__, __LINE__);
1108
1109         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1110
1111         if (read_rt && likely(c.i.rt))
1112                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1113         else if (load_rt)
1114                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1115
1116         if (is_tagged) {
1117                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1118         } else {
1119                 lut_entry = lightrec_get_lut_entry(block);
1120                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1121                                   C_WRAPPER_RW_GENERIC);
1122         }
1123 }
1124
1125 static u32 rec_ram_mask(struct lightrec_state *state)
1126 {
1127         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1128 }
1129
1130 static void rec_store_memory(struct lightrec_cstate *cstate,
1131                              const struct block *block,
1132                              u16 offset, jit_code_t code,
1133                              jit_code_t swap_code,
1134                              uintptr_t addr_offset, u32 addr_mask,
1135                              bool invalidate)
1136 {
1137         const struct lightrec_state *state = cstate->state;
1138         struct regcache *reg_cache = cstate->reg_cache;
1139         struct opcode *op = &block->opcode_list[offset];
1140         jit_state_t *_jit = block->_jit;
1141         union code c = op->c;
1142         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1143         s16 imm = (s16)c.i.imm;
1144         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1145         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1146         bool no_mask = op_flag_no_mask(op->flags);
1147         bool add_imm = c.i.imm &&
1148                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1149                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1150         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1151
1152         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1153         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1154         if (need_tmp)
1155                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1156
1157         addr_reg = rs;
1158
1159         if (add_imm) {
1160                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1161                 addr_reg = tmp;
1162                 imm = 0;
1163         } else if (simm) {
1164                 lut_offt += simm;
1165         }
1166
1167         if (!no_mask) {
1168                 jit_andi(tmp, addr_reg, addr_mask);
1169                 addr_reg = tmp;
1170         }
1171
1172         if (addr_offset) {
1173                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1174                 jit_addi(tmp2, addr_reg, addr_offset);
1175                 addr_reg2 = tmp2;
1176         } else {
1177                 addr_reg2 = addr_reg;
1178         }
1179
1180         if (is_big_endian() && swap_code && c.i.rt) {
1181                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1182
1183                 jit_new_node_ww(swap_code, tmp3, rt);
1184                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1185
1186                 lightrec_free_reg(reg_cache, tmp3);
1187         } else {
1188                 jit_new_node_www(code, imm, addr_reg2, rt);
1189         }
1190
1191         lightrec_free_reg(reg_cache, rt);
1192
1193         if (invalidate) {
1194                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1195
1196                 if (c.i.op != OP_SW) {
1197                         jit_andi(tmp, addr_reg, ~3);
1198                         addr_reg = tmp;
1199                 }
1200
1201                 if (!lut_is_32bit(state)) {
1202                         jit_lshi(tmp, addr_reg, 1);
1203                         addr_reg = tmp;
1204                 }
1205
1206                 if (addr_reg == rs && c.i.rs == 0) {
1207                         addr_reg = LIGHTREC_REG_STATE;
1208                 } else {
1209                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1210                         addr_reg = tmp;
1211                 }
1212
1213                 if (lut_is_32bit(state))
1214                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1215                 else
1216                         jit_stxi(lut_offt, addr_reg, tmp3);
1217
1218                 lightrec_free_reg(reg_cache, tmp3);
1219         }
1220
1221         if (addr_offset)
1222                 lightrec_free_reg(reg_cache, tmp2);
1223         if (need_tmp)
1224                 lightrec_free_reg(reg_cache, tmp);
1225         lightrec_free_reg(reg_cache, rs);
1226 }
1227
1228 static void rec_store_ram(struct lightrec_cstate *cstate,
1229                           const struct block *block,
1230                           u16 offset, jit_code_t code,
1231                           jit_code_t swap_code, bool invalidate)
1232 {
1233         struct lightrec_state *state = cstate->state;
1234
1235         _jit_note(block->_jit, __FILE__, __LINE__);
1236
1237         return rec_store_memory(cstate, block, offset, code, swap_code,
1238                                 state->offset_ram, rec_ram_mask(state),
1239                                 invalidate);
1240 }
1241
1242 static void rec_store_scratch(struct lightrec_cstate *cstate,
1243                               const struct block *block, u16 offset,
1244                               jit_code_t code, jit_code_t swap_code)
1245 {
1246         _jit_note(block->_jit, __FILE__, __LINE__);
1247
1248         return rec_store_memory(cstate, block, offset, code, swap_code,
1249                                 cstate->state->offset_scratch,
1250                                 0x1fffffff, false);
1251 }
1252
1253 static void rec_store_io(struct lightrec_cstate *cstate,
1254                          const struct block *block, u16 offset,
1255                          jit_code_t code, jit_code_t swap_code)
1256 {
1257         _jit_note(block->_jit, __FILE__, __LINE__);
1258
1259         return rec_store_memory(cstate, block, offset, code, swap_code,
1260                                 cstate->state->offset_io,
1261                                 0x1fffffff, false);
1262 }
1263
1264 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1265                                            const struct block *block,
1266                                            u16 offset, jit_code_t code,
1267                                            jit_code_t swap_code)
1268 {
1269         struct lightrec_state *state = cstate->state;
1270         struct regcache *reg_cache = cstate->reg_cache;
1271         union code c = block->opcode_list[offset].c;
1272         jit_state_t *_jit = block->_jit;
1273         jit_node_t *to_not_ram, *to_end;
1274         u8 tmp, tmp2, rs, rt;
1275         s16 imm;
1276
1277         jit_note(__FILE__, __LINE__);
1278         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1279         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1280
1281         if (state->offset_ram || state->offset_scratch)
1282                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1283
1284         /* Convert to KUNSEG and avoid RAM mirrors */
1285         if (state->mirrors_mapped) {
1286                 imm = (s16)c.i.imm;
1287                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1288         } else if (c.i.imm) {
1289                 imm = 0;
1290                 jit_addi(tmp, rs, (s16)c.i.imm);
1291                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1292         } else {
1293                 imm = 0;
1294                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1295         }
1296
1297         lightrec_free_reg(reg_cache, rs);
1298
1299         if (state->offset_ram != state->offset_scratch) {
1300                 to_not_ram = jit_bmsi(tmp, BIT(28));
1301
1302                 jit_movi(tmp2, state->offset_ram);
1303
1304                 to_end = jit_b();
1305                 jit_patch(to_not_ram);
1306
1307                 jit_movi(tmp2, state->offset_scratch);
1308                 jit_patch(to_end);
1309         } else if (state->offset_ram) {
1310                 jit_movi(tmp2, state->offset_ram);
1311         }
1312
1313         if (state->offset_ram || state->offset_scratch) {
1314                 jit_addr(tmp, tmp, tmp2);
1315                 lightrec_free_reg(reg_cache, tmp2);
1316         }
1317
1318         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1319
1320         if (is_big_endian() && swap_code && c.i.rt) {
1321                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1322
1323                 jit_new_node_ww(swap_code, tmp2, rt);
1324                 jit_new_node_www(code, imm, tmp, tmp2);
1325
1326                 lightrec_free_reg(reg_cache, tmp2);
1327         } else {
1328                 jit_new_node_www(code, imm, tmp, rt);
1329         }
1330
1331         lightrec_free_reg(reg_cache, rt);
1332         lightrec_free_reg(reg_cache, tmp);
1333 }
1334
1335 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1336                              u16 offset, jit_code_t code, jit_code_t swap_code)
1337 {
1338         struct lightrec_state *state = cstate->state;
1339         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1340         struct regcache *reg_cache = cstate->reg_cache;
1341         union code c = block->opcode_list[offset].c;
1342         jit_state_t *_jit = block->_jit;
1343         jit_node_t *to_not_ram, *to_end;
1344         u8 tmp, tmp2, tmp3, rs, rt;
1345
1346         jit_note(__FILE__, __LINE__);
1347
1348         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1349         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1350         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1351
1352         /* Convert to KUNSEG and avoid RAM mirrors */
1353         if (c.i.imm) {
1354                 jit_addi(tmp2, rs, (s16)c.i.imm);
1355                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1356         } else {
1357                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1358         }
1359
1360         lightrec_free_reg(reg_cache, rs);
1361         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1362
1363         to_not_ram = jit_bgti(tmp2, ram_size);
1364
1365         /* Compute the offset to the code LUT */
1366         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1367         if (!lut_is_32bit(state))
1368                 jit_lshi(tmp, tmp, 1);
1369         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1370
1371         /* Write NULL to the code LUT to invalidate any block that's there */
1372         if (lut_is_32bit(state))
1373                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1374         else
1375                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1376
1377         if (state->offset_ram != state->offset_scratch) {
1378                 jit_movi(tmp, state->offset_ram);
1379
1380                 to_end = jit_b();
1381         }
1382
1383         jit_patch(to_not_ram);
1384
1385         if (state->offset_ram || state->offset_scratch)
1386                 jit_movi(tmp, state->offset_scratch);
1387
1388         if (state->offset_ram != state->offset_scratch)
1389                 jit_patch(to_end);
1390
1391         if (state->offset_ram || state->offset_scratch)
1392                 jit_addr(tmp2, tmp2, tmp);
1393
1394         lightrec_free_reg(reg_cache, tmp);
1395         lightrec_free_reg(reg_cache, tmp3);
1396
1397         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1398
1399         if (is_big_endian() && swap_code && c.i.rt) {
1400                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1401
1402                 jit_new_node_ww(swap_code, tmp, rt);
1403                 jit_new_node_www(code, 0, tmp2, tmp);
1404
1405                 lightrec_free_reg(reg_cache, tmp);
1406         } else {
1407                 jit_new_node_www(code, 0, tmp2, rt);
1408         }
1409
1410         lightrec_free_reg(reg_cache, rt);
1411         lightrec_free_reg(reg_cache, tmp2);
1412 }
1413
1414 static void rec_store(struct lightrec_cstate *state,
1415                       const struct block *block, u16 offset,
1416                       jit_code_t code, jit_code_t swap_code)
1417 {
1418         u32 flags = block->opcode_list[offset].flags;
1419         bool no_invalidate = op_flag_no_invalidate(flags) ||
1420                 state->state->invalidate_from_dma_only;
1421
1422         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1423         case LIGHTREC_IO_RAM:
1424                 rec_store_ram(state, block, offset, code,
1425                               swap_code, !no_invalidate);
1426                 break;
1427         case LIGHTREC_IO_SCRATCH:
1428                 rec_store_scratch(state, block, offset, code, swap_code);
1429                 break;
1430         case LIGHTREC_IO_DIRECT:
1431                 if (no_invalidate) {
1432                         rec_store_direct_no_invalidate(state, block, offset,
1433                                                        code, swap_code);
1434                 } else {
1435                         rec_store_direct(state, block, offset, code, swap_code);
1436                 }
1437                 break;
1438         case LIGHTREC_IO_DIRECT_HW:
1439                 rec_store_io(state, block, offset, code, swap_code);
1440                 break;
1441         default:
1442                 rec_io(state, block, offset, true, false);
1443                 break;
1444         }
1445 }
1446
1447 static void rec_SB(struct lightrec_cstate *state,
1448                    const struct block *block, u16 offset)
1449 {
1450         _jit_name(block->_jit, __func__);
1451         rec_store(state, block, offset, jit_code_stxi_c, 0);
1452 }
1453
1454 static void rec_SH(struct lightrec_cstate *state,
1455                    const struct block *block, u16 offset)
1456 {
1457         _jit_name(block->_jit, __func__);
1458         rec_store(state, block, offset,
1459                   jit_code_stxi_s, jit_code_bswapr_us);
1460 }
1461
1462 static void rec_SW(struct lightrec_cstate *state,
1463                    const struct block *block, u16 offset)
1464
1465 {
1466         _jit_name(block->_jit, __func__);
1467         rec_store(state, block, offset,
1468                   jit_code_stxi_i, jit_code_bswapr_ui);
1469 }
1470
1471 static void rec_SWL(struct lightrec_cstate *state,
1472                     const struct block *block, u16 offset)
1473 {
1474         _jit_name(block->_jit, __func__);
1475         rec_io(state, block, offset, true, false);
1476 }
1477
1478 static void rec_SWR(struct lightrec_cstate *state,
1479                     const struct block *block, u16 offset)
1480 {
1481         _jit_name(block->_jit, __func__);
1482         rec_io(state, block, offset, true, false);
1483 }
1484
1485 static void rec_SWC2(struct lightrec_cstate *state,
1486                      const struct block *block, u16 offset)
1487 {
1488         _jit_name(block->_jit, __func__);
1489         rec_io(state, block, offset, false, false);
1490 }
1491
1492 static void rec_load_memory(struct lightrec_cstate *cstate,
1493                             const struct block *block, u16 offset,
1494                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1495                             uintptr_t addr_offset, u32 addr_mask)
1496 {
1497         struct regcache *reg_cache = cstate->reg_cache;
1498         struct opcode *op = &block->opcode_list[offset];
1499         jit_state_t *_jit = block->_jit;
1500         u8 rs, rt, addr_reg, flags = REG_EXT;
1501         bool no_mask = op_flag_no_mask(op->flags);
1502         union code c = op->c;
1503         s16 imm;
1504
1505         if (!c.i.rt)
1506                 return;
1507
1508         if (is_unsigned)
1509                 flags |= REG_ZEXT;
1510
1511         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1512         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1513
1514         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1515                 jit_addi(rt, rs, (s16)c.i.imm);
1516                 addr_reg = rt;
1517                 imm = 0;
1518         } else {
1519                 addr_reg = rs;
1520                 imm = (s16)c.i.imm;
1521         }
1522
1523         if (!no_mask) {
1524                 jit_andi(rt, addr_reg, addr_mask);
1525                 addr_reg = rt;
1526         }
1527
1528         if (addr_offset) {
1529                 jit_addi(rt, addr_reg, addr_offset);
1530                 addr_reg = rt;
1531         }
1532
1533         jit_new_node_www(code, rt, addr_reg, imm);
1534
1535         if (is_big_endian() && swap_code) {
1536                 jit_new_node_ww(swap_code, rt, rt);
1537
1538                 if (c.i.op == OP_LH)
1539                         jit_extr_s(rt, rt);
1540                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1541                         jit_extr_i(rt, rt);
1542         }
1543
1544         lightrec_free_reg(reg_cache, rs);
1545         lightrec_free_reg(reg_cache, rt);
1546 }
1547
1548 static void rec_load_ram(struct lightrec_cstate *cstate,
1549                          const struct block *block, u16 offset,
1550                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1551 {
1552         _jit_note(block->_jit, __FILE__, __LINE__);
1553
1554         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1555                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1556 }
1557
1558 static void rec_load_bios(struct lightrec_cstate *cstate,
1559                           const struct block *block, u16 offset,
1560                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1561 {
1562         _jit_note(block->_jit, __FILE__, __LINE__);
1563
1564         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1565                         cstate->state->offset_bios, 0x1fffffff);
1566 }
1567
1568 static void rec_load_scratch(struct lightrec_cstate *cstate,
1569                              const struct block *block, u16 offset,
1570                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1571 {
1572         _jit_note(block->_jit, __FILE__, __LINE__);
1573
1574         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1575                         cstate->state->offset_scratch, 0x1fffffff);
1576 }
1577
1578 static void rec_load_io(struct lightrec_cstate *cstate,
1579                         const struct block *block, u16 offset,
1580                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1581 {
1582         _jit_note(block->_jit, __FILE__, __LINE__);
1583
1584         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1585                         cstate->state->offset_io, 0x1fffffff);
1586 }
1587
1588 static void rec_load_direct(struct lightrec_cstate *cstate,
1589                             const struct block *block, u16 offset,
1590                             jit_code_t code, jit_code_t swap_code,
1591                             bool is_unsigned)
1592 {
1593         struct lightrec_state *state = cstate->state;
1594         struct regcache *reg_cache = cstate->reg_cache;
1595         union code c = block->opcode_list[offset].c;
1596         jit_state_t *_jit = block->_jit;
1597         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1598         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1599         s16 imm;
1600
1601         if (!c.i.rt)
1602                 return;
1603
1604         if (is_unsigned)
1605                 flags |= REG_ZEXT;
1606
1607         jit_note(__FILE__, __LINE__);
1608         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1609         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1610
1611         if ((state->offset_ram == state->offset_bios &&
1612             state->offset_ram == state->offset_scratch &&
1613             state->mirrors_mapped) || !c.i.imm) {
1614                 addr_reg = rs;
1615                 imm = (s16)c.i.imm;
1616         } else {
1617                 jit_addi(rt, rs, (s16)c.i.imm);
1618                 addr_reg = rt;
1619                 imm = 0;
1620
1621                 if (c.i.rs != c.i.rt)
1622                         lightrec_free_reg(reg_cache, rs);
1623         }
1624
1625         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1626
1627         if (state->offset_ram == state->offset_bios &&
1628             state->offset_ram == state->offset_scratch) {
1629                 if (!state->mirrors_mapped) {
1630                         jit_andi(tmp, addr_reg, BIT(28));
1631                         jit_rshi_u(tmp, tmp, 28 - 22);
1632                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1633                         jit_andr(rt, addr_reg, tmp);
1634                 } else {
1635                         jit_andi(rt, addr_reg, 0x1fffffff);
1636                 }
1637
1638                 if (state->offset_ram)
1639                         jit_movi(tmp, state->offset_ram);
1640         } else {
1641                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1642
1643                 /* Convert to KUNSEG and avoid RAM mirrors */
1644                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1645
1646                 if (state->offset_ram)
1647                         jit_movi(tmp, state->offset_ram);
1648
1649                 to_end = jit_b();
1650
1651                 jit_patch(to_not_ram);
1652
1653                 if (state->offset_bios != state->offset_scratch)
1654                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1655
1656                 /* Convert to KUNSEG */
1657                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1658
1659                 jit_movi(tmp, state->offset_bios);
1660
1661                 if (state->offset_bios != state->offset_scratch) {
1662                         to_end2 = jit_b();
1663
1664                         jit_patch(to_not_bios);
1665
1666                         /* Convert to KUNSEG */
1667                         jit_andi(rt, addr_reg, 0x1f800fff);
1668
1669                         if (state->offset_scratch)
1670                                 jit_movi(tmp, state->offset_scratch);
1671
1672                         jit_patch(to_end2);
1673                 }
1674
1675                 jit_patch(to_end);
1676         }
1677
1678         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1679                 jit_addr(rt, rt, tmp);
1680
1681         jit_new_node_www(code, rt, rt, imm);
1682
1683         if (is_big_endian() && swap_code) {
1684                 jit_new_node_ww(swap_code, rt, rt);
1685
1686                 if (c.i.op == OP_LH)
1687                         jit_extr_s(rt, rt);
1688                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1689                         jit_extr_i(rt, rt);
1690         }
1691
1692         lightrec_free_reg(reg_cache, addr_reg);
1693         lightrec_free_reg(reg_cache, rt);
1694         lightrec_free_reg(reg_cache, tmp);
1695 }
1696
1697 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1698                      u16 offset, jit_code_t code, jit_code_t swap_code,
1699                      bool is_unsigned)
1700 {
1701         u32 flags = block->opcode_list[offset].flags;
1702
1703         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1704         case LIGHTREC_IO_RAM:
1705                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1706                 break;
1707         case LIGHTREC_IO_BIOS:
1708                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1709                 break;
1710         case LIGHTREC_IO_SCRATCH:
1711                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1712                 break;
1713         case LIGHTREC_IO_DIRECT_HW:
1714                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1715                 break;
1716         case LIGHTREC_IO_DIRECT:
1717                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1718                 break;
1719         default:
1720                 rec_io(state, block, offset, false, true);
1721                 break;
1722         }
1723 }
1724
1725 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1726 {
1727         _jit_name(block->_jit, __func__);
1728         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1729 }
1730
1731 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1732 {
1733         _jit_name(block->_jit, __func__);
1734         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1735 }
1736
1737 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1738 {
1739         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1740
1741         _jit_name(block->_jit, __func__);
1742         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1743 }
1744
1745 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1746 {
1747         _jit_name(block->_jit, __func__);
1748         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1749 }
1750
1751 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1752 {
1753         _jit_name(block->_jit, __func__);
1754         rec_io(state, block, offset, true, true);
1755 }
1756
1757 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1758 {
1759         _jit_name(block->_jit, __func__);
1760         rec_io(state, block, offset, true, true);
1761 }
1762
1763 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1764 {
1765         jit_code_t code;
1766
1767         if (is_big_endian() && __WORDSIZE == 64)
1768                 code = jit_code_ldxi_ui;
1769         else
1770                 code = jit_code_ldxi_i;
1771
1772         _jit_name(block->_jit, __func__);
1773         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1774 }
1775
1776 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1777 {
1778         _jit_name(block->_jit, __func__);
1779         rec_io(state, block, offset, false, false);
1780 }
1781
1782 static void rec_break_syscall(struct lightrec_cstate *state,
1783                               const struct block *block, u16 offset,
1784                               u32 exit_code)
1785 {
1786         struct regcache *reg_cache = state->reg_cache;
1787         jit_state_t *_jit = block->_jit;
1788         u8 tmp;
1789
1790         _jit_note(block->_jit, __FILE__, __LINE__);
1791
1792         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1793
1794         jit_movi(tmp, exit_code);
1795         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1796                    LIGHTREC_REG_STATE, tmp);
1797
1798         lightrec_free_reg(reg_cache, tmp);
1799
1800         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1801         lightrec_emit_end_of_block(state, block, offset, -1,
1802                                    get_ds_pc(block, offset, 0),
1803                                    31, 0, true);
1804 }
1805
1806 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1807                                 const struct block *block, u16 offset)
1808 {
1809         _jit_name(block->_jit, __func__);
1810         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1811 }
1812
1813 static void rec_special_BREAK(struct lightrec_cstate *state,
1814                               const struct block *block, u16 offset)
1815 {
1816         _jit_name(block->_jit, __func__);
1817         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1818 }
1819
1820 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1821 {
1822         struct regcache *reg_cache = state->reg_cache;
1823         union code c = block->opcode_list[offset].c;
1824         jit_state_t *_jit = block->_jit;
1825
1826         jit_note(__FILE__, __LINE__);
1827         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1828         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1829
1830         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1831
1832         if (c.i.op == OP_CP0 &&
1833             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1834             (c.r.rd == 12 || c.r.rd == 13))
1835                 lightrec_emit_end_of_block(state, block, offset, -1,
1836                                            get_ds_pc(block, offset, 1),
1837                                            0, 0, true);
1838 }
1839
1840 static void
1841 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1842 {
1843         struct regcache *reg_cache = state->reg_cache;
1844         union code c = block->opcode_list[offset].c;
1845         jit_state_t *_jit = block->_jit;
1846         u8 rt;
1847
1848         jit_note(__FILE__, __LINE__);
1849
1850         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1851
1852         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1853                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1854
1855         lightrec_free_reg(reg_cache, rt);
1856 }
1857
1858 static bool block_in_bios(const struct lightrec_cstate *state,
1859                           const struct block *block)
1860 {
1861         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1862         u32 pc = kunseg(block->pc);
1863
1864         return pc >= bios->pc && pc < bios->pc + bios->length;
1865 }
1866
1867 static void
1868 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1869 {
1870         struct regcache *reg_cache = state->reg_cache;
1871         const union code c = block->opcode_list[offset].c;
1872         jit_state_t *_jit = block->_jit;
1873         u8 rt, tmp = 0, tmp2, status;
1874
1875         jit_note(__FILE__, __LINE__);
1876
1877         switch(c.r.rd) {
1878         case 1:
1879         case 4:
1880         case 8:
1881         case 14:
1882         case 15:
1883                 /* Those registers are read-only */
1884                 return;
1885         default:
1886                 break;
1887         }
1888
1889         if (block_in_bios(state, block) && c.r.rd == 12) {
1890                 /* If we are running code from the BIOS, handle writes to the
1891                  * Status register in C. BIOS code may toggle bit 16 which will
1892                  * map/unmap the RAM, while game code cannot do that. */
1893                 rec_mtc(state, block, offset);
1894                 return;
1895         }
1896
1897         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1898
1899         if (c.r.rd != 13) {
1900                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1901                            LIGHTREC_REG_STATE, rt);
1902         }
1903
1904         if (c.r.rd == 12 || c.r.rd == 13) {
1905                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1906                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1907                            offsetof(struct lightrec_state, regs.cp0[13]));
1908
1909                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1910         }
1911
1912         if (c.r.rd == 12) {
1913                 status = rt;
1914         } else if (c.r.rd == 13) {
1915                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1916                 jit_andi(tmp2, rt, 0x0300);
1917                 jit_ori(tmp, tmp, 0x0300);
1918                 jit_xori(tmp, tmp, 0x0300);
1919                 jit_orr(tmp, tmp, tmp2);
1920                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1921                            offsetof(struct lightrec_state, regs.cp0[12]));
1922                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1923                            LIGHTREC_REG_STATE, tmp);
1924                 status = tmp2;
1925         }
1926
1927         if (c.r.rd == 12 || c.r.rd == 13) {
1928                 /* Exit dynarec in case there's a software interrupt.
1929                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1930                 jit_andr(tmp, tmp, status);
1931                 jit_andi(tmp, tmp, 0x0300);
1932                 jit_nei(tmp, tmp, 0);
1933                 jit_andr(tmp, tmp, status);
1934         }
1935
1936         if (c.r.rd == 12) {
1937                 /* Exit dynarec in case we unmask a hardware interrupt.
1938                  * exit_flags = !(~status & 0x401) */
1939
1940                 jit_comr(tmp2, status);
1941                 jit_andi(tmp2, tmp2, 0x401);
1942                 jit_eqi(tmp2, tmp2, 0);
1943                 jit_orr(tmp, tmp, tmp2);
1944         }
1945
1946         if (c.r.rd == 12 || c.r.rd == 13) {
1947                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1948                            LIGHTREC_REG_STATE, tmp);
1949
1950                 lightrec_free_reg(reg_cache, tmp);
1951                 lightrec_free_reg(reg_cache, tmp2);
1952         }
1953
1954         lightrec_free_reg(reg_cache, rt);
1955
1956         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1957             (c.r.rd == 12 || c.r.rd == 13))
1958                 lightrec_emit_eob(state, block, offset + 1, true);
1959 }
1960
1961 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1962                          const struct block *block, u16 offset)
1963 {
1964         _jit_name(block->_jit, __func__);
1965         rec_mfc0(state, block, offset);
1966 }
1967
1968 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1969                          const struct block *block, u16 offset)
1970 {
1971         _jit_name(block->_jit, __func__);
1972         rec_mfc0(state, block, offset);
1973 }
1974
1975 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1976                          const struct block *block, u16 offset)
1977 {
1978         _jit_name(block->_jit, __func__);
1979         rec_mtc0(state, block, offset);
1980 }
1981
1982 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1983                          const struct block *block, u16 offset)
1984 {
1985         _jit_name(block->_jit, __func__);
1986         rec_mtc0(state, block, offset);
1987 }
1988
1989 static unsigned int cp2d_i_offset(u8 reg)
1990 {
1991         return offsetof(struct lightrec_state, regs.cp2d[reg]);
1992 }
1993
1994 static unsigned int cp2d_s_offset(u8 reg)
1995 {
1996         return cp2d_i_offset(reg) + is_big_endian() * 2;
1997 }
1998
1999 static unsigned int cp2c_i_offset(u8 reg)
2000 {
2001         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2002 }
2003
2004 static unsigned int cp2c_s_offset(u8 reg)
2005 {
2006         return cp2c_i_offset(reg) + is_big_endian() * 2;
2007 }
2008
2009 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2010                                const struct block *block, u16 offset)
2011 {
2012         struct regcache *reg_cache = state->reg_cache;
2013         const union code c = block->opcode_list[offset].c;
2014         jit_state_t *_jit = block->_jit;
2015         const u32 zext_regs = 0x300f0080;
2016         u8 rt, tmp, tmp2, tmp3, out, flags;
2017         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2018         unsigned int i;
2019
2020         _jit_name(block->_jit, __func__);
2021
2022         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2023         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2024
2025         switch (reg) {
2026         case 1:
2027         case 3:
2028         case 5:
2029         case 8:
2030         case 9:
2031         case 10:
2032         case 11:
2033                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2034                 break;
2035         case 7:
2036         case 16:
2037         case 17:
2038         case 18:
2039         case 19:
2040                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2041                 break;
2042         case 28:
2043         case 29:
2044                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2045                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2046                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2047
2048                 for (i = 0; i < 3; i++) {
2049                         out = i == 0 ? rt : tmp;
2050
2051                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2052                         jit_movi(tmp2, 0x1f);
2053                         jit_rshi(out, tmp, 7);
2054
2055                         jit_ltr(tmp3, tmp2, out);
2056                         jit_movnr(out, tmp2, tmp3);
2057
2058                         jit_gei(tmp2, out, 0);
2059                         jit_movzr(out, tmp2, tmp2);
2060
2061                         if (i > 0) {
2062                                 jit_lshi(tmp, tmp, 5 * i);
2063                                 jit_orr(rt, rt, tmp);
2064                         }
2065                 }
2066
2067
2068                 lightrec_free_reg(reg_cache, tmp);
2069                 lightrec_free_reg(reg_cache, tmp2);
2070                 lightrec_free_reg(reg_cache, tmp3);
2071                 break;
2072         default:
2073                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2074                 break;
2075         }
2076
2077         lightrec_free_reg(reg_cache, rt);
2078 }
2079
2080 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2081                                const struct block *block, u16 offset)
2082 {
2083         struct regcache *reg_cache = state->reg_cache;
2084         const union code c = block->opcode_list[offset].c;
2085         jit_state_t *_jit = block->_jit;
2086         u8 rt;
2087
2088         _jit_name(block->_jit, __func__);
2089
2090         switch (c.r.rd) {
2091         case 4:
2092         case 12:
2093         case 20:
2094         case 26:
2095         case 27:
2096         case 29:
2097         case 30:
2098                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2099                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2100                 break;
2101         default:
2102                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2103                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2104                 break;
2105         }
2106
2107         lightrec_free_reg(reg_cache, rt);
2108 }
2109
2110 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2111                                const struct block *block, u16 offset)
2112 {
2113         struct regcache *reg_cache = state->reg_cache;
2114         const union code c = block->opcode_list[offset].c;
2115         jit_state_t *_jit = block->_jit;
2116         jit_node_t *loop, *to_loop;
2117         u8 rt, tmp, tmp2, flags = 0;
2118
2119         _jit_name(block->_jit, __func__);
2120
2121         if (c.r.rd == 31)
2122                 return;
2123
2124         if (c.r.rd == 30)
2125                 flags |= REG_EXT;
2126
2127         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2128
2129         switch (c.r.rd) {
2130         case 15:
2131                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2132                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2133
2134                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2135                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2136
2137                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2138                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2139                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2140
2141                 lightrec_free_reg(reg_cache, tmp);
2142                 lightrec_free_reg(reg_cache, tmp2);
2143                 break;
2144         case 28:
2145                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2146
2147                 jit_lshi(tmp, rt, 7);
2148                 jit_andi(tmp, tmp, 0xf80);
2149                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2150
2151                 jit_lshi(tmp, rt, 2);
2152                 jit_andi(tmp, tmp, 0xf80);
2153                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2154
2155                 jit_rshi(tmp, rt, 3);
2156                 jit_andi(tmp, tmp, 0xf80);
2157                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2158
2159                 lightrec_free_reg(reg_cache, tmp);
2160                 break;
2161         case 30:
2162                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2163                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2164
2165                 /* if (rt < 0) rt = ~rt; */
2166                 jit_rshi(tmp, rt, 31);
2167                 jit_xorr(tmp, rt, tmp);
2168
2169                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2170                 jit_lshi(tmp, tmp, 1);
2171                 jit_movi(tmp2, 33);
2172
2173                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2174                 loop = jit_label();
2175                 jit_subi(tmp2, tmp2, 1);
2176                 jit_rshi_u(tmp, tmp, 1);
2177                 to_loop = jit_bnei(tmp, 0);
2178
2179                 jit_patch_at(to_loop, loop);
2180
2181                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2182                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2183
2184                 lightrec_free_reg(reg_cache, tmp);
2185                 lightrec_free_reg(reg_cache, tmp2);
2186                 break;
2187         default:
2188                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2189                 break;
2190         }
2191
2192         lightrec_free_reg(reg_cache, rt);
2193 }
2194
2195 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2196                                const struct block *block, u16 offset)
2197 {
2198         struct regcache *reg_cache = state->reg_cache;
2199         const union code c = block->opcode_list[offset].c;
2200         jit_state_t *_jit = block->_jit;
2201         u8 rt, tmp, tmp2;
2202
2203         _jit_name(block->_jit, __func__);
2204
2205         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2206
2207         switch (c.r.rd) {
2208         case 4:
2209         case 12:
2210         case 20:
2211         case 26:
2212         case 27:
2213         case 29:
2214         case 30:
2215                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2216                 break;
2217         case 31:
2218                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2219                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2220
2221                 jit_andi(tmp, rt, 0x7f87e000);
2222                 jit_nei(tmp, tmp, 0);
2223                 jit_lshi(tmp, tmp, 31);
2224
2225                 jit_andi(tmp2, rt, 0x7ffff000);
2226                 jit_orr(tmp, tmp2, tmp);
2227
2228                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2229
2230                 lightrec_free_reg(reg_cache, tmp);
2231                 lightrec_free_reg(reg_cache, tmp2);
2232                 break;
2233
2234         default:
2235                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2236         }
2237
2238         lightrec_free_reg(reg_cache, rt);
2239 }
2240
2241 static void rec_cp0_RFE(struct lightrec_cstate *state,
2242                         const struct block *block, u16 offset)
2243 {
2244         struct regcache *reg_cache = state->reg_cache;
2245         jit_state_t *_jit = block->_jit;
2246         u8 status, tmp;
2247
2248         jit_name(__func__);
2249         jit_note(__FILE__, __LINE__);
2250
2251         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2252         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2253                    offsetof(struct lightrec_state, regs.cp0[12]));
2254
2255         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2256
2257         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2258         jit_rshi(tmp, status, 2);
2259         jit_andi(tmp, tmp, 0xf);
2260         jit_andi(status, status, ~0xful);
2261         jit_orr(status, status, tmp);
2262
2263         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2264                    offsetof(struct lightrec_state, regs.cp0[13]));
2265         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2266                    LIGHTREC_REG_STATE, status);
2267
2268         /* Exit dynarec in case there's a software interrupt.
2269          * exit_flags = !!(status & cause & 0x0300) & status; */
2270         jit_andr(tmp, tmp, status);
2271         jit_andi(tmp, tmp, 0x0300);
2272         jit_nei(tmp, tmp, 0);
2273         jit_andr(tmp, tmp, status);
2274         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2275                    LIGHTREC_REG_STATE, tmp);
2276
2277         lightrec_free_reg(reg_cache, status);
2278         lightrec_free_reg(reg_cache, tmp);
2279 }
2280
2281 static void rec_CP(struct lightrec_cstate *state,
2282                    const struct block *block, u16 offset)
2283 {
2284         union code c = block->opcode_list[offset].c;
2285         jit_state_t *_jit = block->_jit;
2286
2287         jit_name(__func__);
2288         jit_note(__FILE__, __LINE__);
2289
2290         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2291 }
2292
2293 static void rec_meta_MOV(struct lightrec_cstate *state,
2294                          const struct block *block, u16 offset)
2295 {
2296         struct regcache *reg_cache = state->reg_cache;
2297         union code c = block->opcode_list[offset].c;
2298         jit_state_t *_jit = block->_jit;
2299         u8 rs, rd;
2300
2301         _jit_name(block->_jit, __func__);
2302         jit_note(__FILE__, __LINE__);
2303         if (c.r.rs)
2304                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2305         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2306
2307         if (c.r.rs == 0)
2308                 jit_movi(rd, 0);
2309         else
2310                 jit_extr_i(rd, rs);
2311
2312         if (c.r.rs)
2313                 lightrec_free_reg(reg_cache, rs);
2314         lightrec_free_reg(reg_cache, rd);
2315 }
2316
2317 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2318                                const struct block *block,
2319                                u16 offset)
2320 {
2321         struct regcache *reg_cache = state->reg_cache;
2322         union code c = block->opcode_list[offset].c;
2323         jit_state_t *_jit = block->_jit;
2324         u8 rs, rt;
2325
2326         _jit_name(block->_jit, __func__);
2327         jit_note(__FILE__, __LINE__);
2328
2329         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2330         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2331
2332         if (c.i.op == OP_META_EXTC)
2333                 jit_extr_c(rt, rs);
2334         else
2335                 jit_extr_s(rt, rs);
2336
2337         lightrec_free_reg(reg_cache, rs);
2338         lightrec_free_reg(reg_cache, rt);
2339 }
2340
2341 static void rec_meta_MULT2(struct lightrec_cstate *state,
2342                            const struct block *block,
2343                            u16 offset)
2344 {
2345         struct regcache *reg_cache = state->reg_cache;
2346         union code c = block->opcode_list[offset].c;
2347         jit_state_t *_jit = block->_jit;
2348         u8 reg_lo = get_mult_div_lo(c);
2349         u8 reg_hi = get_mult_div_hi(c);
2350         u32 flags = block->opcode_list[offset].flags;
2351         bool is_signed = c.i.op == OP_META_MULT2;
2352         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2353
2354         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2355                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2356                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2357         }
2358
2359         _jit_name(block->_jit, __func__);
2360         jit_note(__FILE__, __LINE__);
2361
2362         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2363
2364         if (!op_flag_no_lo(flags)) {
2365                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2366
2367                 if (c.r.op < 32)
2368                         jit_lshi(lo, rs, c.r.op);
2369                 else
2370                         jit_movi(lo, 0);
2371
2372                 lightrec_free_reg(reg_cache, lo);
2373         }
2374
2375         if (!op_flag_no_hi(flags)) {
2376                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags);
2377
2378                 if (c.r.op >= 32)
2379                         jit_lshi(hi, rs, c.r.op - 32);
2380                 else if (is_signed)
2381                         jit_rshi(hi, rs, 32 - c.r.op);
2382                 else
2383                         jit_rshi_u(hi, rs, 32 - c.r.op);
2384
2385                 lightrec_free_reg(reg_cache, hi);
2386         }
2387
2388         lightrec_free_reg(reg_cache, rs);
2389
2390         _jit_name(block->_jit, __func__);
2391         jit_note(__FILE__, __LINE__);
2392 }
2393
2394 static const lightrec_rec_func_t rec_standard[64] = {
2395         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2396         [OP_SPECIAL]            = rec_SPECIAL,
2397         [OP_REGIMM]             = rec_REGIMM,
2398         [OP_J]                  = rec_J,
2399         [OP_JAL]                = rec_JAL,
2400         [OP_BEQ]                = rec_BEQ,
2401         [OP_BNE]                = rec_BNE,
2402         [OP_BLEZ]               = rec_BLEZ,
2403         [OP_BGTZ]               = rec_BGTZ,
2404         [OP_ADDI]               = rec_ADDI,
2405         [OP_ADDIU]              = rec_ADDIU,
2406         [OP_SLTI]               = rec_SLTI,
2407         [OP_SLTIU]              = rec_SLTIU,
2408         [OP_ANDI]               = rec_ANDI,
2409         [OP_ORI]                = rec_ORI,
2410         [OP_XORI]               = rec_XORI,
2411         [OP_LUI]                = rec_LUI,
2412         [OP_CP0]                = rec_CP0,
2413         [OP_CP2]                = rec_CP2,
2414         [OP_LB]                 = rec_LB,
2415         [OP_LH]                 = rec_LH,
2416         [OP_LWL]                = rec_LWL,
2417         [OP_LW]                 = rec_LW,
2418         [OP_LBU]                = rec_LBU,
2419         [OP_LHU]                = rec_LHU,
2420         [OP_LWR]                = rec_LWR,
2421         [OP_SB]                 = rec_SB,
2422         [OP_SH]                 = rec_SH,
2423         [OP_SWL]                = rec_SWL,
2424         [OP_SW]                 = rec_SW,
2425         [OP_SWR]                = rec_SWR,
2426         [OP_LWC2]               = rec_LWC2,
2427         [OP_SWC2]               = rec_SWC2,
2428
2429         [OP_META_MOV]           = rec_meta_MOV,
2430         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2431         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2432         [OP_META_MULT2]         = rec_meta_MULT2,
2433         [OP_META_MULTU2]        = rec_meta_MULT2,
2434 };
2435
2436 static const lightrec_rec_func_t rec_special[64] = {
2437         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2438         [OP_SPECIAL_SLL]        = rec_special_SLL,
2439         [OP_SPECIAL_SRL]        = rec_special_SRL,
2440         [OP_SPECIAL_SRA]        = rec_special_SRA,
2441         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2442         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2443         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2444         [OP_SPECIAL_JR]         = rec_special_JR,
2445         [OP_SPECIAL_JALR]       = rec_special_JALR,
2446         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2447         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2448         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2449         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2450         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2451         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2452         [OP_SPECIAL_MULT]       = rec_special_MULT,
2453         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2454         [OP_SPECIAL_DIV]        = rec_special_DIV,
2455         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2456         [OP_SPECIAL_ADD]        = rec_special_ADD,
2457         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2458         [OP_SPECIAL_SUB]        = rec_special_SUB,
2459         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2460         [OP_SPECIAL_AND]        = rec_special_AND,
2461         [OP_SPECIAL_OR]         = rec_special_OR,
2462         [OP_SPECIAL_XOR]        = rec_special_XOR,
2463         [OP_SPECIAL_NOR]        = rec_special_NOR,
2464         [OP_SPECIAL_SLT]        = rec_special_SLT,
2465         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2466 };
2467
2468 static const lightrec_rec_func_t rec_regimm[64] = {
2469         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2470         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2471         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2472         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2473         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2474 };
2475
2476 static const lightrec_rec_func_t rec_cp0[64] = {
2477         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2478         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2479         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2480         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2481         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2482         [OP_CP0_RFE]            = rec_cp0_RFE,
2483 };
2484
2485 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2486         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2487         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2488         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2489         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2490         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2491 };
2492
2493 static void rec_SPECIAL(struct lightrec_cstate *state,
2494                         const struct block *block, u16 offset)
2495 {
2496         union code c = block->opcode_list[offset].c;
2497         lightrec_rec_func_t f = rec_special[c.r.op];
2498
2499         if (!HAS_DEFAULT_ELM && unlikely(!f))
2500                 unknown_opcode(state, block, offset);
2501         else
2502                 (*f)(state, block, offset);
2503 }
2504
2505 static void rec_REGIMM(struct lightrec_cstate *state,
2506                        const struct block *block, u16 offset)
2507 {
2508         union code c = block->opcode_list[offset].c;
2509         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2510
2511         if (!HAS_DEFAULT_ELM && unlikely(!f))
2512                 unknown_opcode(state, block, offset);
2513         else
2514                 (*f)(state, block, offset);
2515 }
2516
2517 static void rec_CP0(struct lightrec_cstate *state,
2518                     const struct block *block, u16 offset)
2519 {
2520         union code c = block->opcode_list[offset].c;
2521         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2522
2523         if (!HAS_DEFAULT_ELM && unlikely(!f))
2524                 rec_CP(state, block, offset);
2525         else
2526                 (*f)(state, block, offset);
2527 }
2528
2529 static void rec_CP2(struct lightrec_cstate *state,
2530                     const struct block *block, u16 offset)
2531 {
2532         union code c = block->opcode_list[offset].c;
2533
2534         if (c.r.op == OP_CP2_BASIC) {
2535                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2536
2537                 if (HAS_DEFAULT_ELM || likely(f)) {
2538                         (*f)(state, block, offset);
2539                         return;
2540                 }
2541         }
2542
2543         rec_CP(state, block, offset);
2544 }
2545
2546 void lightrec_rec_opcode(struct lightrec_cstate *state,
2547                          const struct block *block, u16 offset)
2548 {
2549         struct regcache *reg_cache = state->reg_cache;
2550         struct lightrec_branch_target *target;
2551         const struct opcode *op = &block->opcode_list[offset];
2552         jit_state_t *_jit = block->_jit;
2553         lightrec_rec_func_t f;
2554         u16 unload_offset;
2555
2556         if (op_flag_sync(op->flags)) {
2557                 if (state->cycles)
2558                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2559                 state->cycles = 0;
2560
2561                 lightrec_storeback_regs(reg_cache, _jit);
2562                 lightrec_regcache_reset(reg_cache);
2563
2564                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2565                 target = &state->targets[state->nb_targets++];
2566                 target->offset = offset;
2567                 target->label = jit_indirect();
2568         }
2569
2570         if (likely(op->opcode)) {
2571                 f = rec_standard[op->i.op];
2572
2573                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2574                         unknown_opcode(state, block, offset);
2575                 else
2576                         (*f)(state, block, offset);
2577         }
2578
2579         if (OPT_EARLY_UNLOAD) {
2580                 unload_offset = offset +
2581                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2582
2583                 lightrec_do_early_unload(state, block, unload_offset);
2584         }
2585 }