gpu_neon: don't include vector_ops.h in the main header
[pcsx_rearmed.git] / deps / lightrec / emitter.c
1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 /*
3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
4  */
5
6 #include "blockcache.h"
7 #include "debug.h"
8 #include "disassembler.h"
9 #include "emitter.h"
10 #include "lightning-wrapper.h"
11 #include "optimizer.h"
12 #include "regcache.h"
13
14 #include <stdbool.h>
15 #include <stddef.h>
16
17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
18
19 /* Forward declarations */
20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
24
25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
26 {
27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
28                 block->opcode_list[offset].c.opcode,
29                 block->pc + (offset << 2));
30 }
31
32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
33                                        const struct block *block, u16 offset,
34                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
35                                        u32 link, bool update_cycles)
36 {
37         struct regcache *reg_cache = state->reg_cache;
38         jit_state_t *_jit = block->_jit;
39         const struct opcode *op = &block->opcode_list[offset],
40                             *next = &block->opcode_list[offset + 1];
41         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
42         u16 offset_after_eob;
43
44         jit_note(__FILE__, __LINE__);
45
46         if (link) {
47                 /* Update the $ra register */
48                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
49                 jit_movi(link_reg, link);
50                 lightrec_free_reg(reg_cache, link_reg);
51         }
52
53         if (reg_new_pc < 0) {
54                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
55                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
56
57                 jit_movi(reg_new_pc, imm);
58         }
59
60         if (has_delay_slot(op->c) &&
61             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
62                 cycles += lightrec_cycles_of_opcode(next->c);
63
64                 /* Recompile the delay slot */
65                 if (next->c.opcode)
66                         lightrec_rec_opcode(state, block, offset + 1);
67         }
68
69         /* Clean the remaining registers */
70         lightrec_clean_regs(reg_cache, _jit);
71
72         jit_movr(JIT_V0, reg_new_pc);
73
74         if (cycles && update_cycles) {
75                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
76                 pr_debug("EOB: %u cycles\n", cycles);
77         }
78
79         offset_after_eob = offset + 1 +
80                 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
81
82         if (offset_after_eob < block->nb_ops)
83                 state->branches[state->nb_branches++] = jit_b();
84 }
85
86 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
87                        u16 offset, bool after_op)
88 {
89         struct regcache *reg_cache = state->reg_cache;
90         jit_state_t *_jit = block->_jit;
91         union code c = block->opcode_list[offset].c;
92         u32 cycles = state->cycles;
93
94         if (after_op)
95                 cycles += lightrec_cycles_of_opcode(c);
96
97         lightrec_clean_regs(reg_cache, _jit);
98
99         jit_movi(JIT_V0, block->pc + (offset << 2));
100         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
101
102         state->branches[state->nb_branches++] = jit_b();
103 }
104
105 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
106 {
107         struct regcache *reg_cache = state->reg_cache;
108         jit_state_t *_jit = block->_jit;
109         const struct opcode *op = &block->opcode_list[offset];
110         u8 rs;
111
112         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
113         lightrec_lock_reg(reg_cache, _jit, rs);
114
115         return rs;
116 }
117
118 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
119 {
120         u8 rs = get_jr_jalr_reg(state, block, offset);
121
122         _jit_name(block->_jit, __func__);
123         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
124 }
125
126 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
127 {
128         u8 rs = get_jr_jalr_reg(state, block, offset);
129         union code c = block->opcode_list[offset].c;
130
131         _jit_name(block->_jit, __func__);
132         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
133                                    get_branch_pc(block, offset, 2), true);
134 }
135
136 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
137 {
138         union code c = block->opcode_list[offset].c;
139
140         _jit_name(block->_jit, __func__);
141         lightrec_emit_end_of_block(state, block, offset, -1,
142                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
143                                    31, 0, true);
144 }
145
146 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
147 {
148         union code c = block->opcode_list[offset].c;
149
150         _jit_name(block->_jit, __func__);
151         lightrec_emit_end_of_block(state, block, offset, -1,
152                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
153                                    31, get_branch_pc(block, offset, 2), true);
154 }
155
156 static void lightrec_do_early_unload(struct lightrec_cstate *state,
157                                      const struct block *block, u16 offset)
158 {
159         struct regcache *reg_cache = state->reg_cache;
160         const struct opcode *op = &block->opcode_list[offset];
161         jit_state_t *_jit = block->_jit;
162         unsigned int i;
163         u8 reg;
164         struct {
165                 u8 reg, op;
166         } reg_ops[3] = {
167                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
168                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
169                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
170         };
171
172         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
173                 reg = reg_ops[i].reg;
174
175                 switch (reg_ops[i].op) {
176                 case LIGHTREC_REG_UNLOAD:
177                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
178                         break;
179
180                 case LIGHTREC_REG_DISCARD:
181                         lightrec_discard_reg_if_loaded(reg_cache, reg);
182                         break;
183
184                 case LIGHTREC_REG_CLEAN:
185                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
186                         break;
187                 default:
188                         break;
189                 };
190         }
191 }
192
193 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
194                   jit_code_t code, u32 link, bool unconditional, bool bz)
195 {
196         struct regcache *reg_cache = state->reg_cache;
197         struct native_register *regs_backup;
198         jit_state_t *_jit = block->_jit;
199         struct lightrec_branch *branch;
200         const struct opcode *op = &block->opcode_list[offset],
201                             *next = &block->opcode_list[offset + 1];
202         jit_node_t *addr;
203         u8 link_reg, rs, rt;
204         bool is_forward = (s16)op->i.imm >= -1;
205         int op_cycles = lightrec_cycles_of_opcode(op->c);
206         u32 target_offset, cycles = state->cycles + op_cycles;
207         u32 next_pc;
208
209         jit_note(__FILE__, __LINE__);
210
211         if (!op_flag_no_ds(op->flags))
212                 cycles += lightrec_cycles_of_opcode(next->c);
213
214         state->cycles = -op_cycles;
215
216         if (!unconditional) {
217                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
218                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
219                                                     _jit, op->i.rt, REG_EXT);
220
221                 /* Unload dead registers before evaluating the branch */
222                 if (OPT_EARLY_UNLOAD)
223                         lightrec_do_early_unload(state, block, offset);
224         }
225
226         if (cycles)
227                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
228
229         if (!unconditional) {
230                 /* Generate the branch opcode */
231                 addr = jit_new_node_pww(code, NULL, rs, rt);
232
233                 lightrec_free_regs(reg_cache);
234                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
235         }
236
237         if (op_flag_local_branch(op->flags)) {
238                 /* Recompile the delay slot */
239                 if (next && next->opcode && !op_flag_no_ds(op->flags))
240                         lightrec_rec_opcode(state, block, offset + 1);
241
242                 if (link) {
243                         /* Update the $ra register */
244                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
245                         jit_movi(link_reg, link);
246                         lightrec_free_reg(reg_cache, link_reg);
247                 }
248
249                 /* Clean remaining registers */
250                 lightrec_clean_regs(reg_cache, _jit);
251
252                 target_offset = offset + 1 + (s16)op->i.imm
253                         - !!op_flag_no_ds(op->flags);
254                 pr_debug("Adding local branch to offset 0x%x\n",
255                          target_offset << 2);
256                 branch = &state->local_branches[
257                         state->nb_local_branches++];
258
259                 branch->target = target_offset;
260                 if (is_forward)
261                         branch->branch = jit_b();
262                 else
263                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
264         }
265
266         if (!op_flag_local_branch(op->flags) || !is_forward) {
267                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
268                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
269                                            31, link, false);
270         }
271
272         if (!unconditional) {
273                 jit_patch(addr);
274                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
275
276                 if (bz && link) {
277                         /* Update the $ra register */
278                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
279                                                           31, REG_EXT);
280                         jit_movi(link_reg, (s32)link);
281                         lightrec_free_reg(reg_cache, link_reg);
282                 }
283
284                 if (!op_flag_no_ds(op->flags) && next->opcode)
285                         lightrec_rec_opcode(state, block, offset + 1);
286         }
287 }
288
289 static void rec_BNE(struct lightrec_cstate *state,
290                     const struct block *block, u16 offset)
291 {
292         union code c = block->opcode_list[offset].c;
293
294         _jit_name(block->_jit, __func__);
295
296         if (c.i.rt == 0)
297                 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
298         else
299                 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
300 }
301
302 static void rec_BEQ(struct lightrec_cstate *state,
303                     const struct block *block, u16 offset)
304 {
305         union code c = block->opcode_list[offset].c;
306
307         _jit_name(block->_jit, __func__);
308
309         if (c.i.rt == 0)
310                 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
311         else
312                 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
313 }
314
315 static void rec_BLEZ(struct lightrec_cstate *state,
316                      const struct block *block, u16 offset)
317 {
318         union code c = block->opcode_list[offset].c;
319
320         _jit_name(block->_jit, __func__);
321         rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
322 }
323
324 static void rec_BGTZ(struct lightrec_cstate *state,
325                      const struct block *block, u16 offset)
326 {
327         _jit_name(block->_jit, __func__);
328         rec_b(state, block, offset, jit_code_blei, 0, false, true);
329 }
330
331 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
332                             const struct block *block, u16 offset)
333 {
334         _jit_name(block->_jit, __func__);
335         rec_b(state, block, offset, jit_code_bgei, 0, false, true);
336 }
337
338 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
339                               const struct block *block, u16 offset)
340 {
341         _jit_name(block->_jit, __func__);
342         rec_b(state, block, offset, jit_code_bgei,
343               get_branch_pc(block, offset, 2), false, true);
344 }
345
346 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
347                             const struct block *block, u16 offset)
348 {
349         union code c = block->opcode_list[offset].c;
350
351         _jit_name(block->_jit, __func__);
352         rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
353 }
354
355 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
356                               const struct block *block, u16 offset)
357 {
358         const struct opcode *op = &block->opcode_list[offset];
359         _jit_name(block->_jit, __func__);
360         rec_b(state, block, offset, jit_code_blti,
361               get_branch_pc(block, offset, 2),
362               !op->i.rs, true);
363 }
364
365 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
366                         u16 offset, jit_code_t code, bool slti)
367 {
368         struct regcache *reg_cache = state->reg_cache;
369         union code c = block->opcode_list[offset].c;
370         jit_state_t *_jit = block->_jit;
371         u8 rs, rt, out_flags = REG_EXT;
372
373         if (slti)
374                 out_flags |= REG_ZEXT;
375
376         jit_note(__FILE__, __LINE__);
377         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
378         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
379
380         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
381
382         lightrec_free_reg(reg_cache, rs);
383         lightrec_free_reg(reg_cache, rt);
384 }
385
386 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
387                             u16 offset, jit_code_t code, bool out_ext)
388 {
389         struct regcache *reg_cache = state->reg_cache;
390         union code c = block->opcode_list[offset].c;
391         jit_state_t *_jit = block->_jit;
392         u8 rd, rt, rs;
393
394         jit_note(__FILE__, __LINE__);
395         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
396         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
397         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
398                                     out_ext ? REG_EXT | REG_ZEXT : 0);
399
400         jit_new_node_www(code, rd, rs, rt);
401
402         lightrec_free_reg(reg_cache, rs);
403         lightrec_free_reg(reg_cache, rt);
404         lightrec_free_reg(reg_cache, rd);
405 }
406
407 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
408                            u16 offset, jit_code_t code)
409 {
410         struct regcache *reg_cache = state->reg_cache;
411         union code c = block->opcode_list[offset].c;
412         jit_state_t *_jit = block->_jit;
413         u8 rd, rt, rs, temp, flags = 0;
414
415         jit_note(__FILE__, __LINE__);
416         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
417
418         if (code == jit_code_rshr)
419                 flags = REG_EXT;
420         else if (code == jit_code_rshr_u)
421                 flags = REG_ZEXT;
422
423         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
424         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
425
426         if (rs != rd && rt != rd) {
427                 jit_andi(rd, rs, 0x1f);
428                 jit_new_node_www(code, rd, rt, rd);
429         } else {
430                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
431                 jit_andi(temp, rs, 0x1f);
432                 jit_new_node_www(code, rd, rt, temp);
433                 lightrec_free_reg(reg_cache, temp);
434         }
435
436         lightrec_free_reg(reg_cache, rs);
437         lightrec_free_reg(reg_cache, rt);
438         lightrec_free_reg(reg_cache, rd);
439 }
440
441 static void rec_movi(struct lightrec_cstate *state,
442                      const struct block *block, u16 offset)
443 {
444         struct regcache *reg_cache = state->reg_cache;
445         union code c = block->opcode_list[offset].c;
446         jit_state_t *_jit = block->_jit;
447         u16 flags = REG_EXT;
448         u8 rt;
449
450         if (!(c.i.imm & 0x8000))
451                 flags |= REG_ZEXT;
452
453         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
454
455         jit_movi(rt, (s32)(s16) c.i.imm);
456
457         lightrec_free_reg(reg_cache, rt);
458 }
459
460 static void rec_ADDIU(struct lightrec_cstate *state,
461                       const struct block *block, u16 offset)
462 {
463         _jit_name(block->_jit, __func__);
464
465         if (block->opcode_list[offset].c.i.rs)
466                 rec_alu_imm(state, block, offset, jit_code_addi, false);
467         else
468                 rec_movi(state, block, offset);
469 }
470
471 static void rec_ADDI(struct lightrec_cstate *state,
472                      const struct block *block, u16 offset)
473 {
474         /* TODO: Handle the exception? */
475         _jit_name(block->_jit, __func__);
476         rec_ADDIU(state, block, offset);
477 }
478
479 static void rec_SLTIU(struct lightrec_cstate *state,
480                       const struct block *block, u16 offset)
481 {
482         _jit_name(block->_jit, __func__);
483         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
484 }
485
486 static void rec_SLTI(struct lightrec_cstate *state,
487                      const struct block *block, u16 offset)
488 {
489         _jit_name(block->_jit, __func__);
490         rec_alu_imm(state, block, offset, jit_code_lti, true);
491 }
492
493 static void rec_ANDI(struct lightrec_cstate *state,
494                      const struct block *block, u16 offset)
495 {
496         struct regcache *reg_cache = state->reg_cache;
497         union code c = block->opcode_list[offset].c;
498         jit_state_t *_jit = block->_jit;
499         u8 rs, rt;
500
501         _jit_name(block->_jit, __func__);
502         jit_note(__FILE__, __LINE__);
503         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
504         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
505                                     REG_EXT | REG_ZEXT);
506
507         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
508          * casts to uint8_t / uint16_t. */
509         if (c.i.imm == 0xff)
510                 jit_extr_uc(rt, rs);
511         else if (c.i.imm == 0xffff)
512                 jit_extr_us(rt, rs);
513         else
514                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
515
516         lightrec_free_reg(reg_cache, rs);
517         lightrec_free_reg(reg_cache, rt);
518 }
519
520 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
521                            u16 offset, jit_code_t code)
522 {
523         struct regcache *reg_cache = state->reg_cache;
524         union code c = block->opcode_list[offset].c;
525         jit_state_t *_jit = block->_jit;
526         u8 rs, rt, flags;
527
528         jit_note(__FILE__, __LINE__);
529         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
530         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
531
532         flags = lightrec_get_reg_in_flags(reg_cache, rs);
533         lightrec_set_reg_out_flags(reg_cache, rt, flags);
534
535         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
536
537         lightrec_free_reg(reg_cache, rs);
538         lightrec_free_reg(reg_cache, rt);
539 }
540
541
542 static void rec_ORI(struct lightrec_cstate *state,
543                     const struct block *block, u16 offset)
544 {
545         _jit_name(block->_jit, __func__);
546         rec_alu_or_xor(state, block, offset, jit_code_ori);
547 }
548
549 static void rec_XORI(struct lightrec_cstate *state,
550                      const struct block *block, u16 offset)
551 {
552         _jit_name(block->_jit, __func__);
553         rec_alu_or_xor(state, block, offset, jit_code_xori);
554 }
555
556 static void rec_LUI(struct lightrec_cstate *state,
557                     const struct block *block, u16 offset)
558 {
559         struct regcache *reg_cache = state->reg_cache;
560         union code c = block->opcode_list[offset].c;
561         jit_state_t *_jit = block->_jit;
562         u8 rt, flags = REG_EXT;
563
564         jit_name(__func__);
565         jit_note(__FILE__, __LINE__);
566
567         if (!(c.i.imm & BIT(15)))
568                 flags |= REG_ZEXT;
569
570         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
571
572         jit_movi(rt, (s32)(c.i.imm << 16));
573
574         lightrec_free_reg(reg_cache, rt);
575 }
576
577 static void rec_special_ADDU(struct lightrec_cstate *state,
578                              const struct block *block, u16 offset)
579 {
580         _jit_name(block->_jit, __func__);
581         rec_alu_special(state, block, offset, jit_code_addr, false);
582 }
583
584 static void rec_special_ADD(struct lightrec_cstate *state,
585                             const struct block *block, u16 offset)
586 {
587         /* TODO: Handle the exception? */
588         _jit_name(block->_jit, __func__);
589         rec_alu_special(state, block, offset, jit_code_addr, false);
590 }
591
592 static void rec_special_SUBU(struct lightrec_cstate *state,
593                              const struct block *block, u16 offset)
594 {
595         _jit_name(block->_jit, __func__);
596         rec_alu_special(state, block, offset, jit_code_subr, false);
597 }
598
599 static void rec_special_SUB(struct lightrec_cstate *state,
600                             const struct block *block, u16 offset)
601 {
602         /* TODO: Handle the exception? */
603         _jit_name(block->_jit, __func__);
604         rec_alu_special(state, block, offset, jit_code_subr, false);
605 }
606
607 static void rec_special_AND(struct lightrec_cstate *state,
608                             const struct block *block, u16 offset)
609 {
610         struct regcache *reg_cache = state->reg_cache;
611         union code c = block->opcode_list[offset].c;
612         jit_state_t *_jit = block->_jit;
613         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
614
615         _jit_name(block->_jit, __func__);
616         jit_note(__FILE__, __LINE__);
617         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
618         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
619         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
620
621         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
622         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
623
624         /* Z(rd) = Z(rs) | Z(rt) */
625         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
626
627         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
628         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
629             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
630             (REG_EXT & flags_rs & flags_rt))
631                 flags_rd |= REG_EXT;
632
633         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
634
635         jit_andr(rd, rs, rt);
636
637         lightrec_free_reg(reg_cache, rs);
638         lightrec_free_reg(reg_cache, rt);
639         lightrec_free_reg(reg_cache, rd);
640 }
641
642 static void rec_special_or_nor(struct lightrec_cstate *state,
643                                const struct block *block, u16 offset, bool nor)
644 {
645         struct regcache *reg_cache = state->reg_cache;
646         union code c = block->opcode_list[offset].c;
647         jit_state_t *_jit = block->_jit;
648         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
649
650         jit_note(__FILE__, __LINE__);
651         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
652         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
653         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
654
655         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
656         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
657
658         /* or: Z(rd) = Z(rs) & Z(rt)
659          * nor: Z(rd) = 0 */
660         if (!nor)
661                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
662
663         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
664         if ((REG_EXT & flags_rs & flags_rt) ||
665             (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
666             (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
667                 flags_rd |= REG_EXT;
668
669         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
670
671         jit_orr(rd, rs, rt);
672
673         if (nor)
674                 jit_comr(rd, rd);
675
676         lightrec_free_reg(reg_cache, rs);
677         lightrec_free_reg(reg_cache, rt);
678         lightrec_free_reg(reg_cache, rd);
679 }
680
681 static void rec_special_OR(struct lightrec_cstate *state,
682                            const struct block *block, u16 offset)
683 {
684         _jit_name(block->_jit, __func__);
685         rec_special_or_nor(state, block, offset, false);
686 }
687
688 static void rec_special_NOR(struct lightrec_cstate *state,
689                             const struct block *block, u16 offset)
690 {
691         _jit_name(block->_jit, __func__);
692         rec_special_or_nor(state, block, offset, true);
693 }
694
695 static void rec_special_XOR(struct lightrec_cstate *state,
696                             const struct block *block, u16 offset)
697 {
698         struct regcache *reg_cache = state->reg_cache;
699         union code c = block->opcode_list[offset].c;
700         jit_state_t *_jit = block->_jit;
701         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
702
703         _jit_name(block->_jit, __func__);
704
705         jit_note(__FILE__, __LINE__);
706         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
707         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
708         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
709
710         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
711         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
712
713         /* Z(rd) = Z(rs) & Z(rt) */
714         flags_rd = REG_ZEXT & flags_rs & flags_rt;
715
716         /* E(rd) = E(rs) & E(rt) */
717         flags_rd |= REG_EXT & flags_rs & flags_rt;
718
719         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
720
721         jit_xorr(rd, rs, rt);
722
723         lightrec_free_reg(reg_cache, rs);
724         lightrec_free_reg(reg_cache, rt);
725         lightrec_free_reg(reg_cache, rd);
726 }
727
728 static void rec_special_SLTU(struct lightrec_cstate *state,
729                              const struct block *block, u16 offset)
730 {
731         _jit_name(block->_jit, __func__);
732         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
733 }
734
735 static void rec_special_SLT(struct lightrec_cstate *state,
736                             const struct block *block, u16 offset)
737 {
738         _jit_name(block->_jit, __func__);
739         rec_alu_special(state, block, offset, jit_code_ltr, true);
740 }
741
742 static void rec_special_SLLV(struct lightrec_cstate *state,
743                              const struct block *block, u16 offset)
744 {
745         _jit_name(block->_jit, __func__);
746         rec_alu_shiftv(state, block, offset, jit_code_lshr);
747 }
748
749 static void rec_special_SRLV(struct lightrec_cstate *state,
750                              const struct block *block, u16 offset)
751 {
752         _jit_name(block->_jit, __func__);
753         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
754 }
755
756 static void rec_special_SRAV(struct lightrec_cstate *state,
757                              const struct block *block, u16 offset)
758 {
759         _jit_name(block->_jit, __func__);
760         rec_alu_shiftv(state, block, offset, jit_code_rshr);
761 }
762
763 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
764                           u16 offset, jit_code_t code)
765 {
766         struct regcache *reg_cache = state->reg_cache;
767         union code c = block->opcode_list[offset].c;
768         jit_state_t *_jit = block->_jit;
769         u8 rd, rt, flags = 0;
770
771         jit_note(__FILE__, __LINE__);
772
773         if (code == jit_code_rshi)
774                 flags = REG_EXT;
775         else if (code == jit_code_rshi_u)
776                 flags = REG_ZEXT;
777
778         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
779
780         /* Input reg is zero-extended, if we SRL at least by one bit, we know
781          * the output reg will be both zero-extended and sign-extended. */
782         if (code == jit_code_rshi_u && c.r.imm)
783                 flags |= REG_EXT;
784         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
785
786         jit_new_node_www(code, rd, rt, c.r.imm);
787
788         lightrec_free_reg(reg_cache, rt);
789         lightrec_free_reg(reg_cache, rd);
790 }
791
792 static void rec_special_SLL(struct lightrec_cstate *state,
793                             const struct block *block, u16 offset)
794 {
795         _jit_name(block->_jit, __func__);
796         rec_alu_shift(state, block, offset, jit_code_lshi);
797 }
798
799 static void rec_special_SRL(struct lightrec_cstate *state,
800                             const struct block *block, u16 offset)
801 {
802         _jit_name(block->_jit, __func__);
803         rec_alu_shift(state, block, offset, jit_code_rshi_u);
804 }
805
806 static void rec_special_SRA(struct lightrec_cstate *state,
807                             const struct block *block, u16 offset)
808 {
809         _jit_name(block->_jit, __func__);
810         rec_alu_shift(state, block, offset, jit_code_rshi);
811 }
812
813 static void rec_alu_mult(struct lightrec_cstate *state,
814                          const struct block *block, u16 offset, bool is_signed)
815 {
816         struct regcache *reg_cache = state->reg_cache;
817         union code c = block->opcode_list[offset].c;
818         u32 flags = block->opcode_list[offset].flags;
819         u8 reg_lo = get_mult_div_lo(c);
820         u8 reg_hi = get_mult_div_hi(c);
821         jit_state_t *_jit = block->_jit;
822         u8 lo, hi, rs, rt, rflags = 0;
823
824         jit_note(__FILE__, __LINE__);
825
826         if (is_signed)
827                 rflags = REG_EXT;
828         else
829                 rflags = REG_ZEXT;
830
831         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
832         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
833
834         if (!op_flag_no_lo(flags))
835                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
836         else if (__WORDSIZE == 32)
837                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
838
839         if (!op_flag_no_hi(flags))
840                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
841
842         if (__WORDSIZE == 32) {
843                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
844                  * operation if the MULT was detected a 32-bit only. */
845                 if (!op_flag_no_hi(flags)) {
846                         if (is_signed)
847                                 jit_qmulr(lo, hi, rs, rt);
848                         else
849                                 jit_qmulr_u(lo, hi, rs, rt);
850                 } else {
851                         jit_mulr(lo, rs, rt);
852                 }
853         } else {
854                 /* On 64-bit systems, do a 64*64->64 bit operation. */
855                 if (op_flag_no_lo(flags)) {
856                         jit_mulr(hi, rs, rt);
857                         jit_rshi(hi, hi, 32);
858                 } else {
859                         jit_mulr(lo, rs, rt);
860
861                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
862                         if (!op_flag_no_hi(flags))
863                                 jit_rshi(hi, lo, 32);
864                 }
865         }
866
867         lightrec_free_reg(reg_cache, rs);
868         lightrec_free_reg(reg_cache, rt);
869         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
870                 lightrec_free_reg(reg_cache, lo);
871         if (!op_flag_no_hi(flags))
872                 lightrec_free_reg(reg_cache, hi);
873 }
874
875 static void rec_alu_div(struct lightrec_cstate *state,
876                         const struct block *block, u16 offset, bool is_signed)
877 {
878         struct regcache *reg_cache = state->reg_cache;
879         union code c = block->opcode_list[offset].c;
880         u32 flags = block->opcode_list[offset].flags;
881         bool no_check = op_flag_no_div_check(flags);
882         u8 reg_lo = get_mult_div_lo(c);
883         u8 reg_hi = get_mult_div_hi(c);
884         jit_state_t *_jit = block->_jit;
885         jit_node_t *branch, *to_end;
886         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
887
888         jit_note(__FILE__, __LINE__);
889
890         if (is_signed)
891                 rflags = REG_EXT;
892         else
893                 rflags = REG_ZEXT;
894
895         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
896         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
897
898         if (!op_flag_no_lo(flags))
899                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
900
901         if (!op_flag_no_hi(flags))
902                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
903
904         /* Jump to special handler if dividing by zero  */
905         if (!no_check)
906                 branch = jit_beqi(rt, 0);
907
908         if (op_flag_no_lo(flags)) {
909                 if (is_signed)
910                         jit_remr(hi, rs, rt);
911                 else
912                         jit_remr_u(hi, rs, rt);
913         } else if (op_flag_no_hi(flags)) {
914                 if (is_signed)
915                         jit_divr(lo, rs, rt);
916                 else
917                         jit_divr_u(lo, rs, rt);
918         } else {
919                 if (is_signed)
920                         jit_qdivr(lo, hi, rs, rt);
921                 else
922                         jit_qdivr_u(lo, hi, rs, rt);
923         }
924
925         if (!no_check) {
926                 /* Jump above the div-by-zero handler */
927                 to_end = jit_b();
928
929                 jit_patch(branch);
930
931                 if (!op_flag_no_lo(flags)) {
932                         if (is_signed) {
933                                 jit_lti(lo, rs, 0);
934                                 jit_lshi(lo, lo, 1);
935                                 jit_subi(lo, lo, 1);
936                         } else {
937                                 jit_movi(lo, 0xffffffff);
938                         }
939                 }
940
941                 if (!op_flag_no_hi(flags))
942                         jit_movr(hi, rs);
943
944                 jit_patch(to_end);
945         }
946
947         lightrec_free_reg(reg_cache, rs);
948         lightrec_free_reg(reg_cache, rt);
949
950         if (!op_flag_no_lo(flags))
951                 lightrec_free_reg(reg_cache, lo);
952
953         if (!op_flag_no_hi(flags))
954                 lightrec_free_reg(reg_cache, hi);
955 }
956
957 static void rec_special_MULT(struct lightrec_cstate *state,
958                              const struct block *block, u16 offset)
959 {
960         _jit_name(block->_jit, __func__);
961         rec_alu_mult(state, block, offset, true);
962 }
963
964 static void rec_special_MULTU(struct lightrec_cstate *state,
965                               const struct block *block, u16 offset)
966 {
967         _jit_name(block->_jit, __func__);
968         rec_alu_mult(state, block, offset, false);
969 }
970
971 static void rec_special_DIV(struct lightrec_cstate *state,
972                             const struct block *block, u16 offset)
973 {
974         _jit_name(block->_jit, __func__);
975         rec_alu_div(state, block, offset, true);
976 }
977
978 static void rec_special_DIVU(struct lightrec_cstate *state,
979                              const struct block *block, u16 offset)
980 {
981         _jit_name(block->_jit, __func__);
982         rec_alu_div(state, block, offset, false);
983 }
984
985 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
986                              const struct block *block, u8 dst, u8 src)
987 {
988         struct regcache *reg_cache = state->reg_cache;
989         jit_state_t *_jit = block->_jit;
990
991         jit_note(__FILE__, __LINE__);
992         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
993         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
994
995         jit_extr_i(dst, src);
996
997         lightrec_free_reg(reg_cache, src);
998         lightrec_free_reg(reg_cache, dst);
999 }
1000
1001 static void rec_special_MFHI(struct lightrec_cstate *state,
1002                              const struct block *block, u16 offset)
1003 {
1004         union code c = block->opcode_list[offset].c;
1005
1006         _jit_name(block->_jit, __func__);
1007         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1008 }
1009
1010 static void rec_special_MTHI(struct lightrec_cstate *state,
1011                              const struct block *block, u16 offset)
1012 {
1013         union code c = block->opcode_list[offset].c;
1014
1015         _jit_name(block->_jit, __func__);
1016         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1017 }
1018
1019 static void rec_special_MFLO(struct lightrec_cstate *state,
1020                              const struct block *block, u16 offset)
1021 {
1022         union code c = block->opcode_list[offset].c;
1023
1024         _jit_name(block->_jit, __func__);
1025         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1026 }
1027
1028 static void rec_special_MTLO(struct lightrec_cstate *state,
1029                              const struct block *block, u16 offset)
1030 {
1031         union code c = block->opcode_list[offset].c;
1032
1033         _jit_name(block->_jit, __func__);
1034         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1035 }
1036
1037 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
1038                               u32 arg, bool with_arg, enum c_wrappers wrapper)
1039 {
1040         struct regcache *reg_cache = state->reg_cache;
1041         jit_state_t *_jit = block->_jit;
1042         u8 tmp;
1043
1044         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1045         jit_ldxi(tmp, LIGHTREC_REG_STATE,
1046                  offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1047
1048         if (with_arg) {
1049                 jit_prepare();
1050                 jit_pushargi(arg);
1051         }
1052
1053         lightrec_regcache_mark_live(reg_cache, _jit);
1054         jit_callr(tmp);
1055
1056         lightrec_free_reg(reg_cache, tmp);
1057         lightrec_regcache_mark_live(reg_cache, _jit);
1058 }
1059
1060 static void rec_io(struct lightrec_cstate *state,
1061                    const struct block *block, u16 offset,
1062                    bool load_rt, bool read_rt)
1063 {
1064         struct regcache *reg_cache = state->reg_cache;
1065         jit_state_t *_jit = block->_jit;
1066         union code c = block->opcode_list[offset].c;
1067         u32 flags = block->opcode_list[offset].flags;
1068         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1069         u32 lut_entry;
1070
1071         jit_note(__FILE__, __LINE__);
1072
1073         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1074
1075         if (read_rt && likely(c.i.rt))
1076                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1077         else if (load_rt)
1078                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1079
1080         if (is_tagged) {
1081                 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1082         } else {
1083                 lut_entry = lightrec_get_lut_entry(block);
1084                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1085                                   true, C_WRAPPER_RW_GENERIC);
1086         }
1087 }
1088
1089 static u32 rec_ram_mask(struct lightrec_state *state)
1090 {
1091         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1092 }
1093
1094 static void rec_store_memory(struct lightrec_cstate *cstate,
1095                              const struct block *block,
1096                              u16 offset, jit_code_t code,
1097                              jit_code_t swap_code,
1098                              uintptr_t addr_offset, u32 addr_mask,
1099                              bool invalidate)
1100 {
1101         const struct lightrec_state *state = cstate->state;
1102         struct regcache *reg_cache = cstate->reg_cache;
1103         struct opcode *op = &block->opcode_list[offset];
1104         jit_state_t *_jit = block->_jit;
1105         union code c = op->c;
1106         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1107         s16 imm = (s16)c.i.imm;
1108         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1109         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1110         bool no_mask = op_flag_no_mask(op->flags);
1111         bool add_imm = c.i.imm &&
1112                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1113                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1114         bool need_tmp = !no_mask || addr_offset || add_imm;
1115         bool need_tmp2 = addr_offset || invalidate;
1116
1117         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1118         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1119         if (need_tmp)
1120                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1121
1122         addr_reg = rs;
1123
1124         if (add_imm) {
1125                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1126                 addr_reg = tmp;
1127                 imm = 0;
1128         } else if (simm) {
1129                 lut_offt += simm;
1130         }
1131
1132         if (!no_mask) {
1133                 jit_andi(tmp, addr_reg, addr_mask);
1134                 addr_reg = tmp;
1135         }
1136
1137         if (need_tmp2)
1138                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1139
1140         if (addr_offset) {
1141                 jit_addi(tmp2, addr_reg, addr_offset);
1142                 addr_reg2 = tmp2;
1143         } else {
1144                 addr_reg2 = addr_reg;
1145         }
1146
1147         if (is_big_endian() && swap_code && c.i.rt) {
1148                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1149
1150                 jit_new_node_ww(swap_code, tmp3, rt);
1151                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1152
1153                 lightrec_free_reg(reg_cache, tmp3);
1154         } else {
1155                 jit_new_node_www(code, imm, addr_reg2, rt);
1156         }
1157
1158         lightrec_free_reg(reg_cache, rt);
1159
1160         if (invalidate) {
1161                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1162
1163                 if (c.i.op != OP_SW) {
1164                         jit_andi(tmp2, addr_reg, ~3);
1165                         addr_reg = tmp2;
1166                 }
1167
1168                 if (!lut_is_32bit(state)) {
1169                         jit_lshi(tmp2, addr_reg, 1);
1170                         addr_reg = tmp2;
1171                 }
1172
1173                 if (addr_reg == rs && c.i.rs == 0) {
1174                         addr_reg = LIGHTREC_REG_STATE;
1175                 } else {
1176                         jit_addr(tmp2, addr_reg, LIGHTREC_REG_STATE);
1177                         addr_reg = tmp2;
1178                 }
1179
1180                 if (lut_is_32bit(state))
1181                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1182                 else
1183                         jit_stxi(lut_offt, addr_reg, tmp3);
1184
1185                 lightrec_free_reg(reg_cache, tmp3);
1186         }
1187
1188         if (need_tmp2)
1189                 lightrec_free_reg(reg_cache, tmp2);
1190         if (need_tmp)
1191                 lightrec_free_reg(reg_cache, tmp);
1192         lightrec_free_reg(reg_cache, rs);
1193 }
1194
1195 static void rec_store_ram(struct lightrec_cstate *cstate,
1196                           const struct block *block,
1197                           u16 offset, jit_code_t code,
1198                           jit_code_t swap_code, bool invalidate)
1199 {
1200         struct lightrec_state *state = cstate->state;
1201
1202         _jit_note(block->_jit, __FILE__, __LINE__);
1203
1204         return rec_store_memory(cstate, block, offset, code, swap_code,
1205                                 state->offset_ram, rec_ram_mask(state),
1206                                 invalidate);
1207 }
1208
1209 static void rec_store_scratch(struct lightrec_cstate *cstate,
1210                               const struct block *block, u16 offset,
1211                               jit_code_t code, jit_code_t swap_code)
1212 {
1213         _jit_note(block->_jit, __FILE__, __LINE__);
1214
1215         return rec_store_memory(cstate, block, offset, code, swap_code,
1216                                 cstate->state->offset_scratch,
1217                                 0x1fffffff, false);
1218 }
1219
1220 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1221                                            const struct block *block,
1222                                            u16 offset, jit_code_t code,
1223                                            jit_code_t swap_code)
1224 {
1225         struct lightrec_state *state = cstate->state;
1226         struct regcache *reg_cache = cstate->reg_cache;
1227         union code c = block->opcode_list[offset].c;
1228         jit_state_t *_jit = block->_jit;
1229         jit_node_t *to_not_ram, *to_end;
1230         u8 tmp, tmp2, rs, rt;
1231         s16 imm;
1232
1233         jit_note(__FILE__, __LINE__);
1234         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1235         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1236         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1237
1238         if (state->offset_ram || state->offset_scratch)
1239                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1240
1241         /* Convert to KUNSEG and avoid RAM mirrors */
1242         if (state->mirrors_mapped) {
1243                 imm = (s16)c.i.imm;
1244                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1245         } else if (c.i.imm) {
1246                 imm = 0;
1247                 jit_addi(tmp, rs, (s16)c.i.imm);
1248                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1249         } else {
1250                 imm = 0;
1251                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1252         }
1253
1254         lightrec_free_reg(reg_cache, rs);
1255
1256         if (state->offset_ram != state->offset_scratch) {
1257                 to_not_ram = jit_bmsi(tmp, BIT(28));
1258
1259                 jit_movi(tmp2, state->offset_ram);
1260
1261                 to_end = jit_b();
1262                 jit_patch(to_not_ram);
1263
1264                 jit_movi(tmp2, state->offset_scratch);
1265                 jit_patch(to_end);
1266         } else if (state->offset_ram) {
1267                 jit_movi(tmp2, state->offset_ram);
1268         }
1269
1270         if (state->offset_ram || state->offset_scratch) {
1271                 jit_addr(tmp, tmp, tmp2);
1272                 lightrec_free_reg(reg_cache, tmp2);
1273         }
1274
1275         if (is_big_endian() && swap_code && c.i.rt) {
1276                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1277
1278                 jit_new_node_ww(swap_code, tmp2, rt);
1279                 jit_new_node_www(code, imm, tmp, tmp2);
1280
1281                 lightrec_free_reg(reg_cache, tmp2);
1282         } else {
1283                 jit_new_node_www(code, imm, tmp, rt);
1284         }
1285
1286         lightrec_free_reg(reg_cache, rt);
1287         lightrec_free_reg(reg_cache, tmp);
1288 }
1289
1290 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1291                              u16 offset, jit_code_t code, jit_code_t swap_code)
1292 {
1293         struct lightrec_state *state = cstate->state;
1294         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1295         struct regcache *reg_cache = cstate->reg_cache;
1296         union code c = block->opcode_list[offset].c;
1297         jit_state_t *_jit = block->_jit;
1298         jit_node_t *to_not_ram, *to_end;
1299         u8 tmp, tmp2, tmp3, rs, rt;
1300
1301         jit_note(__FILE__, __LINE__);
1302
1303         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1304         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1305         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1306
1307         /* Convert to KUNSEG and avoid RAM mirrors */
1308         if (c.i.imm) {
1309                 jit_addi(tmp2, rs, (s16)c.i.imm);
1310                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1311         } else {
1312                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1313         }
1314
1315         lightrec_free_reg(reg_cache, rs);
1316         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1317
1318         to_not_ram = jit_bgti(tmp2, ram_size);
1319
1320         /* Compute the offset to the code LUT */
1321         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1322         if (!lut_is_32bit(state))
1323                 jit_lshi(tmp, tmp, 1);
1324         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1325
1326         /* Write NULL to the code LUT to invalidate any block that's there */
1327         if (lut_is_32bit(state))
1328                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1329         else
1330                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1331
1332         if (state->offset_ram != state->offset_scratch) {
1333                 jit_movi(tmp, state->offset_ram);
1334
1335                 to_end = jit_b();
1336         }
1337
1338         jit_patch(to_not_ram);
1339
1340         if (state->offset_ram || state->offset_scratch)
1341                 jit_movi(tmp, state->offset_scratch);
1342
1343         if (state->offset_ram != state->offset_scratch)
1344                 jit_patch(to_end);
1345
1346         if (state->offset_ram || state->offset_scratch)
1347                 jit_addr(tmp2, tmp2, tmp);
1348
1349         lightrec_free_reg(reg_cache, tmp);
1350         lightrec_free_reg(reg_cache, tmp3);
1351
1352         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1353
1354         if (is_big_endian() && swap_code && c.i.rt) {
1355                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1356
1357                 jit_new_node_ww(swap_code, tmp, rt);
1358                 jit_new_node_www(code, 0, tmp2, tmp);
1359
1360                 lightrec_free_reg(reg_cache, tmp);
1361         } else {
1362                 jit_new_node_www(code, 0, tmp2, rt);
1363         }
1364
1365         lightrec_free_reg(reg_cache, rt);
1366         lightrec_free_reg(reg_cache, tmp2);
1367 }
1368
1369 static void rec_store(struct lightrec_cstate *state,
1370                       const struct block *block, u16 offset,
1371                       jit_code_t code, jit_code_t swap_code)
1372 {
1373         u32 flags = block->opcode_list[offset].flags;
1374         bool no_invalidate = op_flag_no_invalidate(flags) ||
1375                 state->state->invalidate_from_dma_only;
1376
1377         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1378         case LIGHTREC_IO_RAM:
1379                 rec_store_ram(state, block, offset, code,
1380                               swap_code, !no_invalidate);
1381                 break;
1382         case LIGHTREC_IO_SCRATCH:
1383                 rec_store_scratch(state, block, offset, code, swap_code);
1384                 break;
1385         case LIGHTREC_IO_DIRECT:
1386                 if (no_invalidate) {
1387                         rec_store_direct_no_invalidate(state, block, offset,
1388                                                        code, swap_code);
1389                 } else {
1390                         rec_store_direct(state, block, offset, code, swap_code);
1391                 }
1392                 break;
1393         default:
1394                 rec_io(state, block, offset, true, false);
1395                 break;
1396         }
1397 }
1398
1399 static void rec_SB(struct lightrec_cstate *state,
1400                    const struct block *block, u16 offset)
1401 {
1402         _jit_name(block->_jit, __func__);
1403         rec_store(state, block, offset, jit_code_stxi_c, 0);
1404 }
1405
1406 static void rec_SH(struct lightrec_cstate *state,
1407                    const struct block *block, u16 offset)
1408 {
1409         _jit_name(block->_jit, __func__);
1410         rec_store(state, block, offset,
1411                   jit_code_stxi_s, jit_code_bswapr_us);
1412 }
1413
1414 static void rec_SW(struct lightrec_cstate *state,
1415                    const struct block *block, u16 offset)
1416
1417 {
1418         _jit_name(block->_jit, __func__);
1419         rec_store(state, block, offset,
1420                   jit_code_stxi_i, jit_code_bswapr_ui);
1421 }
1422
1423 static void rec_SWL(struct lightrec_cstate *state,
1424                     const struct block *block, u16 offset)
1425 {
1426         _jit_name(block->_jit, __func__);
1427         rec_io(state, block, offset, true, false);
1428 }
1429
1430 static void rec_SWR(struct lightrec_cstate *state,
1431                     const struct block *block, u16 offset)
1432 {
1433         _jit_name(block->_jit, __func__);
1434         rec_io(state, block, offset, true, false);
1435 }
1436
1437 static void rec_SWC2(struct lightrec_cstate *state,
1438                      const struct block *block, u16 offset)
1439 {
1440         _jit_name(block->_jit, __func__);
1441         rec_io(state, block, offset, false, false);
1442 }
1443
1444 static void rec_load_memory(struct lightrec_cstate *cstate,
1445                             const struct block *block, u16 offset,
1446                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1447                             uintptr_t addr_offset, u32 addr_mask)
1448 {
1449         struct regcache *reg_cache = cstate->reg_cache;
1450         struct opcode *op = &block->opcode_list[offset];
1451         jit_state_t *_jit = block->_jit;
1452         u8 rs, rt, addr_reg, flags = REG_EXT;
1453         bool no_mask = op_flag_no_mask(op->flags);
1454         union code c = op->c;
1455         s16 imm;
1456
1457         if (!c.i.rt)
1458                 return;
1459
1460         if (is_unsigned)
1461                 flags |= REG_ZEXT;
1462
1463         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1464         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1465
1466         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1467                 jit_addi(rt, rs, (s16)c.i.imm);
1468                 addr_reg = rt;
1469                 imm = 0;
1470         } else {
1471                 addr_reg = rs;
1472                 imm = (s16)c.i.imm;
1473         }
1474
1475         if (!no_mask) {
1476                 jit_andi(rt, addr_reg, addr_mask);
1477                 addr_reg = rt;
1478         }
1479
1480         if (addr_offset) {
1481                 jit_addi(rt, addr_reg, addr_offset);
1482                 addr_reg = rt;
1483         }
1484
1485         jit_new_node_www(code, rt, addr_reg, imm);
1486
1487         if (is_big_endian() && swap_code) {
1488                 jit_new_node_ww(swap_code, rt, rt);
1489
1490                 if (c.i.op == OP_LH)
1491                         jit_extr_s(rt, rt);
1492                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1493                         jit_extr_i(rt, rt);
1494         }
1495
1496         lightrec_free_reg(reg_cache, rs);
1497         lightrec_free_reg(reg_cache, rt);
1498 }
1499
1500 static void rec_load_ram(struct lightrec_cstate *cstate,
1501                          const struct block *block, u16 offset,
1502                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1503 {
1504         _jit_note(block->_jit, __FILE__, __LINE__);
1505
1506         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1507                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1508 }
1509
1510 static void rec_load_bios(struct lightrec_cstate *cstate,
1511                           const struct block *block, u16 offset,
1512                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1513 {
1514         _jit_note(block->_jit, __FILE__, __LINE__);
1515
1516         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1517                         cstate->state->offset_bios, 0x1fffffff);
1518 }
1519
1520 static void rec_load_scratch(struct lightrec_cstate *cstate,
1521                              const struct block *block, u16 offset,
1522                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1523 {
1524         _jit_note(block->_jit, __FILE__, __LINE__);
1525
1526         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1527                         cstate->state->offset_scratch, 0x1fffffff);
1528 }
1529
1530 static void rec_load_direct(struct lightrec_cstate *cstate,
1531                             const struct block *block, u16 offset,
1532                             jit_code_t code, jit_code_t swap_code,
1533                             bool is_unsigned)
1534 {
1535         struct lightrec_state *state = cstate->state;
1536         struct regcache *reg_cache = cstate->reg_cache;
1537         union code c = block->opcode_list[offset].c;
1538         jit_state_t *_jit = block->_jit;
1539         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1540         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1541         s16 imm;
1542
1543         if (!c.i.rt)
1544                 return;
1545
1546         if (is_unsigned)
1547                 flags |= REG_ZEXT;
1548
1549         jit_note(__FILE__, __LINE__);
1550         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1551         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1552
1553         if ((state->offset_ram == state->offset_bios &&
1554             state->offset_ram == state->offset_scratch &&
1555             state->mirrors_mapped) || !c.i.imm) {
1556                 addr_reg = rs;
1557                 imm = (s16)c.i.imm;
1558         } else {
1559                 jit_addi(rt, rs, (s16)c.i.imm);
1560                 addr_reg = rt;
1561                 imm = 0;
1562
1563                 if (c.i.rs != c.i.rt)
1564                         lightrec_free_reg(reg_cache, rs);
1565         }
1566
1567         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1568
1569         if (state->offset_ram == state->offset_bios &&
1570             state->offset_ram == state->offset_scratch) {
1571                 if (!state->mirrors_mapped) {
1572                         jit_andi(tmp, addr_reg, BIT(28));
1573                         jit_rshi_u(tmp, tmp, 28 - 22);
1574                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1575                         jit_andr(rt, addr_reg, tmp);
1576                 } else {
1577                         jit_andi(rt, addr_reg, 0x1fffffff);
1578                 }
1579
1580                 if (state->offset_ram)
1581                         jit_movi(tmp, state->offset_ram);
1582         } else {
1583                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1584
1585                 /* Convert to KUNSEG and avoid RAM mirrors */
1586                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1587
1588                 if (state->offset_ram)
1589                         jit_movi(tmp, state->offset_ram);
1590
1591                 to_end = jit_b();
1592
1593                 jit_patch(to_not_ram);
1594
1595                 if (state->offset_bios != state->offset_scratch)
1596                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1597
1598                 /* Convert to KUNSEG */
1599                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1600
1601                 jit_movi(tmp, state->offset_bios);
1602
1603                 if (state->offset_bios != state->offset_scratch) {
1604                         to_end2 = jit_b();
1605
1606                         jit_patch(to_not_bios);
1607
1608                         /* Convert to KUNSEG */
1609                         jit_andi(rt, addr_reg, 0x1f800fff);
1610
1611                         if (state->offset_scratch)
1612                                 jit_movi(tmp, state->offset_scratch);
1613
1614                         jit_patch(to_end2);
1615                 }
1616
1617                 jit_patch(to_end);
1618         }
1619
1620         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1621                 jit_addr(rt, rt, tmp);
1622
1623         jit_new_node_www(code, rt, rt, imm);
1624
1625         if (is_big_endian() && swap_code) {
1626                 jit_new_node_ww(swap_code, rt, rt);
1627
1628                 if (c.i.op == OP_LH)
1629                         jit_extr_s(rt, rt);
1630                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1631                         jit_extr_i(rt, rt);
1632         }
1633
1634         lightrec_free_reg(reg_cache, addr_reg);
1635         lightrec_free_reg(reg_cache, rt);
1636         lightrec_free_reg(reg_cache, tmp);
1637 }
1638
1639 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1640                      u16 offset, jit_code_t code, jit_code_t swap_code,
1641                      bool is_unsigned)
1642 {
1643         u32 flags = block->opcode_list[offset].flags;
1644
1645         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1646         case LIGHTREC_IO_RAM:
1647                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1648                 break;
1649         case LIGHTREC_IO_BIOS:
1650                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1651                 break;
1652         case LIGHTREC_IO_SCRATCH:
1653                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1654                 break;
1655         case LIGHTREC_IO_DIRECT:
1656                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1657                 break;
1658         default:
1659                 rec_io(state, block, offset, false, true);
1660                 break;
1661         }
1662 }
1663
1664 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1665 {
1666         _jit_name(block->_jit, __func__);
1667         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1668 }
1669
1670 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1671 {
1672         _jit_name(block->_jit, __func__);
1673         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1674 }
1675
1676 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1677 {
1678         _jit_name(block->_jit, __func__);
1679         rec_load(state, block, offset, jit_code_ldxi_s, jit_code_bswapr_us, false);
1680 }
1681
1682 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1683 {
1684         _jit_name(block->_jit, __func__);
1685         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1686 }
1687
1688 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1689 {
1690         _jit_name(block->_jit, __func__);
1691         rec_io(state, block, offset, true, true);
1692 }
1693
1694 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1695 {
1696         _jit_name(block->_jit, __func__);
1697         rec_io(state, block, offset, true, true);
1698 }
1699
1700 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1701 {
1702         _jit_name(block->_jit, __func__);
1703         rec_load(state, block, offset, jit_code_ldxi_i, jit_code_bswapr_ui, false);
1704 }
1705
1706 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1707 {
1708         _jit_name(block->_jit, __func__);
1709         rec_io(state, block, offset, false, false);
1710 }
1711
1712 static void rec_break_syscall(struct lightrec_cstate *state,
1713                               const struct block *block, u16 offset, bool is_break)
1714 {
1715         _jit_note(block->_jit, __FILE__, __LINE__);
1716
1717         if (is_break)
1718                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1719         else
1720                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1721
1722         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1723         lightrec_emit_end_of_block(state, block, offset, -1,
1724                                    get_ds_pc(block, offset, 0),
1725                                    31, 0, true);
1726 }
1727
1728 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1729                                 const struct block *block, u16 offset)
1730 {
1731         _jit_name(block->_jit, __func__);
1732         rec_break_syscall(state, block, offset, false);
1733 }
1734
1735 static void rec_special_BREAK(struct lightrec_cstate *state,
1736                               const struct block *block, u16 offset)
1737 {
1738         _jit_name(block->_jit, __func__);
1739         rec_break_syscall(state, block, offset, true);
1740 }
1741
1742 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1743 {
1744         struct regcache *reg_cache = state->reg_cache;
1745         union code c = block->opcode_list[offset].c;
1746         jit_state_t *_jit = block->_jit;
1747
1748         jit_note(__FILE__, __LINE__);
1749         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1750         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1751
1752         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1753
1754         if (c.i.op == OP_CP0 &&
1755             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1756             (c.r.rd == 12 || c.r.rd == 13))
1757                 lightrec_emit_end_of_block(state, block, offset, -1,
1758                                            get_ds_pc(block, offset, 1),
1759                                            0, 0, true);
1760 }
1761
1762 static void
1763 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1764 {
1765         struct regcache *reg_cache = state->reg_cache;
1766         union code c = block->opcode_list[offset].c;
1767         jit_state_t *_jit = block->_jit;
1768         u8 rt;
1769
1770         jit_note(__FILE__, __LINE__);
1771
1772         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1773
1774         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1775                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1776
1777         lightrec_free_reg(reg_cache, rt);
1778 }
1779
1780 static bool block_in_bios(const struct lightrec_cstate *state,
1781                           const struct block *block)
1782 {
1783         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1784         u32 pc = kunseg(block->pc);
1785
1786         return pc >= bios->pc && pc < bios->pc + bios->length;
1787 }
1788
1789 static void
1790 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1791 {
1792         struct regcache *reg_cache = state->reg_cache;
1793         const union code c = block->opcode_list[offset].c;
1794         jit_state_t *_jit = block->_jit;
1795         u8 rt, tmp = 0, tmp2, status;
1796
1797         jit_note(__FILE__, __LINE__);
1798
1799         switch(c.r.rd) {
1800         case 1:
1801         case 4:
1802         case 8:
1803         case 14:
1804         case 15:
1805                 /* Those registers are read-only */
1806                 return;
1807         default:
1808                 break;
1809         }
1810
1811         if (block_in_bios(state, block) && c.r.rd == 12) {
1812                 /* If we are running code from the BIOS, handle writes to the
1813                  * Status register in C. BIOS code may toggle bit 16 which will
1814                  * map/unmap the RAM, while game code cannot do that. */
1815                 rec_mtc(state, block, offset);
1816                 return;
1817         }
1818
1819         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1820
1821         if (c.r.rd != 13) {
1822                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1823                            LIGHTREC_REG_STATE, rt);
1824         }
1825
1826         if (c.r.rd == 12 || c.r.rd == 13) {
1827                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1828                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1829                            offsetof(struct lightrec_state, regs.cp0[13]));
1830
1831                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1832         }
1833
1834         if (c.r.rd == 12) {
1835                 status = rt;
1836         } else if (c.r.rd == 13) {
1837                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1838                 jit_andi(tmp2, rt, 0x0300);
1839                 jit_ori(tmp, tmp, 0x0300);
1840                 jit_xori(tmp, tmp, 0x0300);
1841                 jit_orr(tmp, tmp, tmp2);
1842                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1843                            offsetof(struct lightrec_state, regs.cp0[12]));
1844                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1845                            LIGHTREC_REG_STATE, tmp);
1846                 status = tmp2;
1847         }
1848
1849         if (c.r.rd == 12 || c.r.rd == 13) {
1850                 /* Exit dynarec in case there's a software interrupt.
1851                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1852                 jit_andr(tmp, tmp, status);
1853                 jit_andi(tmp, tmp, 0x0300);
1854                 jit_nei(tmp, tmp, 0);
1855                 jit_andr(tmp, tmp, status);
1856         }
1857
1858         if (c.r.rd == 12) {
1859                 /* Exit dynarec in case we unmask a hardware interrupt.
1860                  * exit_flags = !(~status & 0x401) */
1861
1862                 jit_comr(tmp2, status);
1863                 jit_andi(tmp2, tmp2, 0x401);
1864                 jit_eqi(tmp2, tmp2, 0);
1865                 jit_orr(tmp, tmp, tmp2);
1866         }
1867
1868         if (c.r.rd == 12 || c.r.rd == 13) {
1869                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1870                            LIGHTREC_REG_STATE, tmp);
1871
1872                 lightrec_free_reg(reg_cache, tmp);
1873                 lightrec_free_reg(reg_cache, tmp2);
1874         }
1875
1876         lightrec_free_reg(reg_cache, rt);
1877
1878         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1879             (c.r.rd == 12 || c.r.rd == 13))
1880                 lightrec_emit_eob(state, block, offset + 1, true);
1881 }
1882
1883 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1884                          const struct block *block, u16 offset)
1885 {
1886         _jit_name(block->_jit, __func__);
1887         rec_mfc0(state, block, offset);
1888 }
1889
1890 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1891                          const struct block *block, u16 offset)
1892 {
1893         _jit_name(block->_jit, __func__);
1894         rec_mfc0(state, block, offset);
1895 }
1896
1897 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1898                          const struct block *block, u16 offset)
1899 {
1900         _jit_name(block->_jit, __func__);
1901         rec_mtc0(state, block, offset);
1902 }
1903
1904 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1905                          const struct block *block, u16 offset)
1906 {
1907         _jit_name(block->_jit, __func__);
1908         rec_mtc0(state, block, offset);
1909 }
1910
1911 static unsigned int cp2d_i_offset(u8 reg)
1912 {
1913         return offsetof(struct lightrec_state, regs.cp2d[reg]);
1914 }
1915
1916 static unsigned int cp2d_s_offset(u8 reg)
1917 {
1918         return cp2d_i_offset(reg) + is_big_endian() * 2;
1919 }
1920
1921 static unsigned int cp2c_i_offset(u8 reg)
1922 {
1923         return offsetof(struct lightrec_state, regs.cp2c[reg]);
1924 }
1925
1926 static unsigned int cp2c_s_offset(u8 reg)
1927 {
1928         return cp2c_i_offset(reg) + is_big_endian() * 2;
1929 }
1930
1931 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1932                                const struct block *block, u16 offset)
1933 {
1934         struct regcache *reg_cache = state->reg_cache;
1935         const union code c = block->opcode_list[offset].c;
1936         jit_state_t *_jit = block->_jit;
1937         const u32 zext_regs = 0x300f0080;
1938         u8 rt, tmp, tmp2, tmp3, out, flags;
1939         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
1940         unsigned int i;
1941
1942         _jit_name(block->_jit, __func__);
1943
1944         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
1945         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
1946
1947         switch (reg) {
1948         case 1:
1949         case 3:
1950         case 5:
1951         case 8:
1952         case 9:
1953         case 10:
1954         case 11:
1955                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1956                 break;
1957         case 7:
1958         case 16:
1959         case 17:
1960         case 18:
1961         case 19:
1962                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1963                 break;
1964         case 28:
1965         case 29:
1966                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1967                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1968                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1969
1970                 for (i = 0; i < 3; i++) {
1971                         out = i == 0 ? rt : tmp;
1972
1973                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
1974                         jit_movi(tmp2, 0x1f);
1975                         jit_rshi(out, tmp, 7);
1976
1977                         jit_ltr(tmp3, tmp2, out);
1978                         jit_movnr(out, tmp2, tmp3);
1979
1980                         jit_gei(tmp2, out, 0);
1981                         jit_movzr(out, tmp2, tmp2);
1982
1983                         if (i > 0) {
1984                                 jit_lshi(tmp, tmp, 5 * i);
1985                                 jit_orr(rt, rt, tmp);
1986                         }
1987                 }
1988
1989
1990                 lightrec_free_reg(reg_cache, tmp);
1991                 lightrec_free_reg(reg_cache, tmp2);
1992                 lightrec_free_reg(reg_cache, tmp3);
1993                 break;
1994         default:
1995                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
1996                 break;
1997         }
1998
1999         lightrec_free_reg(reg_cache, rt);
2000 }
2001
2002 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2003                                const struct block *block, u16 offset)
2004 {
2005         struct regcache *reg_cache = state->reg_cache;
2006         const union code c = block->opcode_list[offset].c;
2007         jit_state_t *_jit = block->_jit;
2008         u8 rt;
2009
2010         _jit_name(block->_jit, __func__);
2011
2012         switch (c.r.rd) {
2013         case 4:
2014         case 12:
2015         case 20:
2016         case 26:
2017         case 27:
2018         case 29:
2019         case 30:
2020                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2021                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2022                 break;
2023         default:
2024                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2025                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2026                 break;
2027         }
2028
2029         lightrec_free_reg(reg_cache, rt);
2030 }
2031
2032 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2033                                const struct block *block, u16 offset)
2034 {
2035         struct regcache *reg_cache = state->reg_cache;
2036         const union code c = block->opcode_list[offset].c;
2037         jit_state_t *_jit = block->_jit;
2038         jit_node_t *loop, *to_loop;
2039         u8 rt, tmp, tmp2, flags = 0;
2040
2041         _jit_name(block->_jit, __func__);
2042
2043         if (c.r.rd == 31)
2044                 return;
2045
2046         if (c.r.rd == 30)
2047                 flags |= REG_EXT;
2048
2049         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2050
2051         switch (c.r.rd) {
2052         case 15:
2053                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2054                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2055
2056                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2057                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2058
2059                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2060                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2061                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2062
2063                 lightrec_free_reg(reg_cache, tmp);
2064                 lightrec_free_reg(reg_cache, tmp2);
2065                 break;
2066         case 28:
2067                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2068
2069                 jit_lshi(tmp, rt, 7);
2070                 jit_andi(tmp, tmp, 0xf80);
2071                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2072
2073                 jit_lshi(tmp, rt, 2);
2074                 jit_andi(tmp, tmp, 0xf80);
2075                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2076
2077                 jit_rshi(tmp, rt, 3);
2078                 jit_andi(tmp, tmp, 0xf80);
2079                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2080
2081                 lightrec_free_reg(reg_cache, tmp);
2082                 break;
2083         case 30:
2084                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2085                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2086
2087                 /* if (rt < 0) rt = ~rt; */
2088                 jit_rshi(tmp, rt, 31);
2089                 jit_xorr(tmp, rt, tmp);
2090
2091                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2092                 jit_lshi(tmp, tmp, 1);
2093                 jit_movi(tmp2, 33);
2094
2095                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2096                 loop = jit_label();
2097                 jit_subi(tmp2, tmp2, 1);
2098                 jit_rshi_u(tmp, tmp, 1);
2099                 to_loop = jit_bnei(tmp, 0);
2100
2101                 jit_patch_at(to_loop, loop);
2102
2103                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2104                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2105
2106                 lightrec_free_reg(reg_cache, tmp);
2107                 lightrec_free_reg(reg_cache, tmp2);
2108                 break;
2109         default:
2110                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2111                 break;
2112         }
2113
2114         lightrec_free_reg(reg_cache, rt);
2115 }
2116
2117 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2118                                const struct block *block, u16 offset)
2119 {
2120         struct regcache *reg_cache = state->reg_cache;
2121         const union code c = block->opcode_list[offset].c;
2122         jit_state_t *_jit = block->_jit;
2123         u8 rt, tmp, tmp2;
2124
2125         _jit_name(block->_jit, __func__);
2126
2127         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2128
2129         switch (c.r.rd) {
2130         case 4:
2131         case 12:
2132         case 20:
2133         case 26:
2134         case 27:
2135         case 29:
2136         case 30:
2137                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2138                 break;
2139         case 31:
2140                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2141                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2142
2143                 jit_andi(tmp, rt, 0x7f87e000);
2144                 jit_nei(tmp, tmp, 0);
2145                 jit_lshi(tmp, tmp, 31);
2146
2147                 jit_andi(tmp2, rt, 0x7ffff000);
2148                 jit_orr(tmp, tmp2, tmp);
2149
2150                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2151
2152                 lightrec_free_reg(reg_cache, tmp);
2153                 lightrec_free_reg(reg_cache, tmp2);
2154                 break;
2155
2156         default:
2157                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2158         }
2159
2160         lightrec_free_reg(reg_cache, rt);
2161 }
2162
2163 static void rec_cp0_RFE(struct lightrec_cstate *state,
2164                         const struct block *block, u16 offset)
2165 {
2166         struct regcache *reg_cache = state->reg_cache;
2167         jit_state_t *_jit = block->_jit;
2168         u8 status, tmp;
2169
2170         jit_name(__func__);
2171         jit_note(__FILE__, __LINE__);
2172
2173         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2174         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2175                    offsetof(struct lightrec_state, regs.cp0[12]));
2176
2177         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2178
2179         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2180         jit_rshi(tmp, status, 2);
2181         jit_andi(tmp, tmp, 0xf);
2182         jit_andi(status, status, ~0xful);
2183         jit_orr(status, status, tmp);
2184
2185         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2186                    offsetof(struct lightrec_state, regs.cp0[13]));
2187         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2188                    LIGHTREC_REG_STATE, status);
2189
2190         /* Exit dynarec in case there's a software interrupt.
2191          * exit_flags = !!(status & cause & 0x0300) & status; */
2192         jit_andr(tmp, tmp, status);
2193         jit_andi(tmp, tmp, 0x0300);
2194         jit_nei(tmp, tmp, 0);
2195         jit_andr(tmp, tmp, status);
2196         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2197                    LIGHTREC_REG_STATE, tmp);
2198
2199         lightrec_free_reg(reg_cache, status);
2200         lightrec_free_reg(reg_cache, tmp);
2201 }
2202
2203 static void rec_CP(struct lightrec_cstate *state,
2204                    const struct block *block, u16 offset)
2205 {
2206         union code c = block->opcode_list[offset].c;
2207         jit_state_t *_jit = block->_jit;
2208
2209         jit_name(__func__);
2210         jit_note(__FILE__, __LINE__);
2211
2212         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
2213 }
2214
2215 static void rec_meta_MOV(struct lightrec_cstate *state,
2216                          const struct block *block, u16 offset)
2217 {
2218         struct regcache *reg_cache = state->reg_cache;
2219         union code c = block->opcode_list[offset].c;
2220         jit_state_t *_jit = block->_jit;
2221         u8 rs, rd;
2222
2223         _jit_name(block->_jit, __func__);
2224         jit_note(__FILE__, __LINE__);
2225         if (c.r.rs)
2226                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2227         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2228
2229         if (c.r.rs == 0)
2230                 jit_movi(rd, 0);
2231         else
2232                 jit_extr_i(rd, rs);
2233
2234         if (c.r.rs)
2235                 lightrec_free_reg(reg_cache, rs);
2236         lightrec_free_reg(reg_cache, rd);
2237 }
2238
2239 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2240                                const struct block *block,
2241                                u16 offset)
2242 {
2243         struct regcache *reg_cache = state->reg_cache;
2244         union code c = block->opcode_list[offset].c;
2245         jit_state_t *_jit = block->_jit;
2246         u8 rs, rt;
2247
2248         _jit_name(block->_jit, __func__);
2249         jit_note(__FILE__, __LINE__);
2250
2251         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2252         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2253
2254         if (c.i.op == OP_META_EXTC)
2255                 jit_extr_c(rt, rs);
2256         else
2257                 jit_extr_s(rt, rs);
2258
2259         lightrec_free_reg(reg_cache, rs);
2260         lightrec_free_reg(reg_cache, rt);
2261 }
2262
2263 static const lightrec_rec_func_t rec_standard[64] = {
2264         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2265         [OP_SPECIAL]            = rec_SPECIAL,
2266         [OP_REGIMM]             = rec_REGIMM,
2267         [OP_J]                  = rec_J,
2268         [OP_JAL]                = rec_JAL,
2269         [OP_BEQ]                = rec_BEQ,
2270         [OP_BNE]                = rec_BNE,
2271         [OP_BLEZ]               = rec_BLEZ,
2272         [OP_BGTZ]               = rec_BGTZ,
2273         [OP_ADDI]               = rec_ADDI,
2274         [OP_ADDIU]              = rec_ADDIU,
2275         [OP_SLTI]               = rec_SLTI,
2276         [OP_SLTIU]              = rec_SLTIU,
2277         [OP_ANDI]               = rec_ANDI,
2278         [OP_ORI]                = rec_ORI,
2279         [OP_XORI]               = rec_XORI,
2280         [OP_LUI]                = rec_LUI,
2281         [OP_CP0]                = rec_CP0,
2282         [OP_CP2]                = rec_CP2,
2283         [OP_LB]                 = rec_LB,
2284         [OP_LH]                 = rec_LH,
2285         [OP_LWL]                = rec_LWL,
2286         [OP_LW]                 = rec_LW,
2287         [OP_LBU]                = rec_LBU,
2288         [OP_LHU]                = rec_LHU,
2289         [OP_LWR]                = rec_LWR,
2290         [OP_SB]                 = rec_SB,
2291         [OP_SH]                 = rec_SH,
2292         [OP_SWL]                = rec_SWL,
2293         [OP_SW]                 = rec_SW,
2294         [OP_SWR]                = rec_SWR,
2295         [OP_LWC2]               = rec_LWC2,
2296         [OP_SWC2]               = rec_SWC2,
2297
2298         [OP_META_MOV]           = rec_meta_MOV,
2299         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2300         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2301 };
2302
2303 static const lightrec_rec_func_t rec_special[64] = {
2304         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2305         [OP_SPECIAL_SLL]        = rec_special_SLL,
2306         [OP_SPECIAL_SRL]        = rec_special_SRL,
2307         [OP_SPECIAL_SRA]        = rec_special_SRA,
2308         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2309         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2310         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2311         [OP_SPECIAL_JR]         = rec_special_JR,
2312         [OP_SPECIAL_JALR]       = rec_special_JALR,
2313         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2314         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2315         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2316         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2317         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2318         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2319         [OP_SPECIAL_MULT]       = rec_special_MULT,
2320         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2321         [OP_SPECIAL_DIV]        = rec_special_DIV,
2322         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2323         [OP_SPECIAL_ADD]        = rec_special_ADD,
2324         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2325         [OP_SPECIAL_SUB]        = rec_special_SUB,
2326         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2327         [OP_SPECIAL_AND]        = rec_special_AND,
2328         [OP_SPECIAL_OR]         = rec_special_OR,
2329         [OP_SPECIAL_XOR]        = rec_special_XOR,
2330         [OP_SPECIAL_NOR]        = rec_special_NOR,
2331         [OP_SPECIAL_SLT]        = rec_special_SLT,
2332         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2333 };
2334
2335 static const lightrec_rec_func_t rec_regimm[64] = {
2336         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2337         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2338         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2339         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2340         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2341 };
2342
2343 static const lightrec_rec_func_t rec_cp0[64] = {
2344         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2345         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2346         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2347         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2348         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2349         [OP_CP0_RFE]            = rec_cp0_RFE,
2350 };
2351
2352 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2353         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2354         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2355         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2356         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2357         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2358 };
2359
2360 static void rec_SPECIAL(struct lightrec_cstate *state,
2361                         const struct block *block, u16 offset)
2362 {
2363         union code c = block->opcode_list[offset].c;
2364         lightrec_rec_func_t f = rec_special[c.r.op];
2365
2366         if (!HAS_DEFAULT_ELM && unlikely(!f))
2367                 unknown_opcode(state, block, offset);
2368         else
2369                 (*f)(state, block, offset);
2370 }
2371
2372 static void rec_REGIMM(struct lightrec_cstate *state,
2373                        const struct block *block, u16 offset)
2374 {
2375         union code c = block->opcode_list[offset].c;
2376         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2377
2378         if (!HAS_DEFAULT_ELM && unlikely(!f))
2379                 unknown_opcode(state, block, offset);
2380         else
2381                 (*f)(state, block, offset);
2382 }
2383
2384 static void rec_CP0(struct lightrec_cstate *state,
2385                     const struct block *block, u16 offset)
2386 {
2387         union code c = block->opcode_list[offset].c;
2388         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2389
2390         if (!HAS_DEFAULT_ELM && unlikely(!f))
2391                 rec_CP(state, block, offset);
2392         else
2393                 (*f)(state, block, offset);
2394 }
2395
2396 static void rec_CP2(struct lightrec_cstate *state,
2397                     const struct block *block, u16 offset)
2398 {
2399         union code c = block->opcode_list[offset].c;
2400
2401         if (c.r.op == OP_CP2_BASIC) {
2402                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2403
2404                 if (HAS_DEFAULT_ELM || likely(f)) {
2405                         (*f)(state, block, offset);
2406                         return;
2407                 }
2408         }
2409
2410         rec_CP(state, block, offset);
2411 }
2412
2413 void lightrec_rec_opcode(struct lightrec_cstate *state,
2414                          const struct block *block, u16 offset)
2415 {
2416         struct regcache *reg_cache = state->reg_cache;
2417         struct lightrec_branch_target *target;
2418         const struct opcode *op = &block->opcode_list[offset];
2419         jit_state_t *_jit = block->_jit;
2420         lightrec_rec_func_t f;
2421         u16 unload_offset;
2422
2423         if (op_flag_sync(op->flags)) {
2424                 if (state->cycles)
2425                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2426                 state->cycles = 0;
2427
2428                 lightrec_storeback_regs(reg_cache, _jit);
2429                 lightrec_regcache_reset(reg_cache);
2430
2431                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2432                 target = &state->targets[state->nb_targets++];
2433                 target->offset = offset;
2434                 target->label = jit_indirect();
2435         }
2436
2437         if (likely(op->opcode)) {
2438                 f = rec_standard[op->i.op];
2439
2440                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2441                         unknown_opcode(state, block, offset);
2442                 else
2443                         (*f)(state, block, offset);
2444         }
2445
2446         if (OPT_EARLY_UNLOAD) {
2447                 unload_offset = offset +
2448                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2449
2450                 lightrec_do_early_unload(state, block, unload_offset);
2451         }
2452 }