notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #835 from pcercuei/misc-fixes
[pcsx_rearmed.git] / deps / lightrec / emitter.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC		 4 */
5
6#include "blockcache.h"
7#include "debug.h"
8#include "disassembler.h"
9#include "emitter.h"
98fa08a5 10#include "lightning-wrapper.h"
d16005f8
PC		 11#include "optimizer.h"
12#include "regcache.h"
13
d16005f8
PC		 14#include <stdbool.h>
15#include <stddef.h>
16
5459088b
PC		 17#define LIGHTNING_UNALIGNED_32BIT 4
18
98fa08a5 19typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
d16005f8
PC		 20
21/* Forward declarations */
98fa08a5
PC		 22static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
23static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
24static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
25static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
cb72ea13 26static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset);
9259d748
PC		 27static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
28 const struct block *block, u16 offset, u8 reg, u8 in_reg);
29static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
30 const struct block *block, u16 offset,
31 u8 reg, u8 out_reg);
d16005f8 32
ba3814c1 33static void
cb72ea13 34lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void))
ba3814c1
PC		 35{
36 /* Prevent jit_jmpi() from using our cycles register as a temporary */
37 jit_live(LIGHTREC_REG_CYCLE);
38
cb72ea13
PC		 39 jit_patch_abs(jit_jmpi(), fn);
40}
41
42static void
43lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
44{
45 lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func);
46}
47
48static void
49lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit)
50{
51 lightrec_jump_to_fn(_jit, state->state->ds_check_func);
ba3814c1
PC		 52}
53
9259d748
PC		 54static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
55 u8 ra_reg, u32 pc, u32 link)
56{
57 u8 link_reg;
58
59 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
60 lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
61 lightrec_free_reg(reg_cache, link_reg);
62}
63
98fa08a5
PC		 64static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
65 const struct block *block, u16 offset,
d16005f8
PC		 66 s8 reg_new_pc, u32 imm, u8 ra_reg,
67 u32 link, bool update_cycles)
68{
d16005f8 69 struct regcache *reg_cache = state->reg_cache;
d16005f8 70 jit_state_t *_jit = block->_jit;
98fa08a5 71 const struct opcode *op = &block->opcode_list[offset],
cb72ea13 72 *ds = get_delay_slot(block->opcode_list, offset);
684432ad 73 u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c);
d75d83f7 74 bool has_ds = has_delay_slot(op->c);
d16005f8
PC		 75
76 jit_note(__FILE__, __LINE__);
77
9259d748
PC		 78 if (link && ra_reg != reg_new_pc)
79 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
d16005f8 80
9259d748
PC		 81 if (reg_new_pc < 0)
82 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
83 else
84 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
d16005f8 85
9259d748
PC		 86 if (link && ra_reg == reg_new_pc) {
87 /* Handle the special case: JALR $r0, $r0
88 * In that case the target PC should be the old value of the
89 * register. */
90 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
d16005f8
PC		 91 }
92
d75d83f7 93 if (has_ds && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
684432ad 94 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
d16005f8
PC		 95
96 /* Recompile the delay slot */
cb72ea13 97 if (ds->c.opcode)
98fa08a5 98 lightrec_rec_opcode(state, block, offset + 1);
d16005f8
PC		 99 }
100
03535202
PC		 101 /* Clean the remaining registers */
102 lightrec_clean_regs(reg_cache, _jit);
d16005f8 103
d16005f8
PC		 104 if (cycles && update_cycles) {
105 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
106 pr_debug("EOB: %u cycles\n", cycles);
107 }
108
d75d83f7 109 if (has_ds && op_flag_load_delay(ds->flags)
a5a6f7b8 110 && opcode_has_load_delay(ds->c) && !state->no_load_delay) {
cb72ea13
PC		 111 /* If the delay slot is a load opcode, its target register
112 * will be written after the first opcode of the target is
113 * executed. Handle this by jumping to a special section of
114 * the dispatcher. It expects the loaded value to be in
115 * REG_TEMP, and the target register number to be in JIT_V1.*/
116 jit_movi(JIT_V1, ds->c.i.rt);
117
118 lightrec_jump_to_ds_check(state, _jit);
119 } else {
120 lightrec_jump_to_eob(state, _jit);
121 }
0e720fb1
PC		 122
123 lightrec_regcache_reset(reg_cache);
d16005f8
PC		 124}
125
cb72ea13
PC		 126void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state,
127 const struct block *block, u16 offset)
128{
129 struct regcache *reg_cache = state->reg_cache;
130 jit_state_t *_jit = block->_jit;
131
132 lightrec_clean_regs(reg_cache, _jit);
133
134 /* Call the interpreter with the block's address in JIT_V1 and the
135 * PC (which might have an offset) in JIT_V0. */
136 lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
137 block->pc + (offset << 2));
0e720fb1
PC		 138 if (lightrec_store_next_pc()) {
139 jit_stxi_i(offsetof(struct lightrec_state, next_pc),
140 LIGHTREC_REG_STATE, JIT_V0);
141 }
142
cb72ea13
PC		 143 jit_movi(JIT_V1, (uintptr_t)block);
144
145 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
146 lightrec_jump_to_fn(_jit, state->state->interpreter_func);
147}
148
149static void lightrec_emit_eob(struct lightrec_cstate *state,
150 const struct block *block, u16 offset)
d16005f8 151{
d16005f8
PC		 152 struct regcache *reg_cache = state->reg_cache;
153 jit_state_t *_jit = block->_jit;
154
03535202 155 lightrec_clean_regs(reg_cache, _jit);
d16005f8 156
9259d748
PC		 157 lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
158 block->pc + (offset << 2));
0e720fb1
PC		 159 if (lightrec_store_next_pc()) {
160 jit_stxi_i(offsetof(struct lightrec_state, next_pc),
161 LIGHTREC_REG_STATE, JIT_V0);
162 }
163
13b02197 164 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
d16005f8 165
ba3814c1 166 lightrec_jump_to_eob(state, _jit);
d16005f8
PC		 167}
168
98fa08a5 169static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 170{
9259d748 171 union code c = block->opcode_list[offset].c;
d16005f8
PC		 172
173 _jit_name(block->_jit, __func__);
9259d748 174 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
98fa08a5
PC		 175}
176
177static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
178{
98fa08a5
PC		 179 union code c = block->opcode_list[offset].c;
180
181 _jit_name(block->_jit, __func__);
9259d748 182 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
98fa08a5 183 get_branch_pc(block, offset, 2), true);
d16005f8
PC		 184}
185
98fa08a5 186static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 187{
98fa08a5
PC		 188 union code c = block->opcode_list[offset].c;
189
d16005f8 190 _jit_name(block->_jit, __func__);
98fa08a5
PC		 191 lightrec_emit_end_of_block(state, block, offset, -1,
192 (block->pc & 0xf0000000) | (c.j.imm << 2),
193 31, 0, true);
d16005f8
PC		 194}
195
98fa08a5 196static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 197{
98fa08a5
PC		 198 union code c = block->opcode_list[offset].c;
199
d16005f8 200 _jit_name(block->_jit, __func__);
98fa08a5
PC		 201 lightrec_emit_end_of_block(state, block, offset, -1,
202 (block->pc & 0xf0000000) | (c.j.imm << 2),
203 31, get_branch_pc(block, offset, 2), true);
d16005f8
PC		 204}
205
03535202
PC		 206static void lightrec_do_early_unload(struct lightrec_cstate *state,
207 const struct block *block, u16 offset)
208{
209 struct regcache *reg_cache = state->reg_cache;
210 const struct opcode *op = &block->opcode_list[offset];
211 jit_state_t *_jit = block->_jit;
212 unsigned int i;
213 u8 reg;
214 struct {
215 u8 reg, op;
216 } reg_ops[3] = {
217 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
218 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
219 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
220 };
221
222 for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
223 reg = reg_ops[i].reg;
224
225 switch (reg_ops[i].op) {
226 case LIGHTREC_REG_UNLOAD:
227 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
228 break;
229
230 case LIGHTREC_REG_DISCARD:
231 lightrec_discard_reg_if_loaded(reg_cache, reg);
232 break;
233
234 case LIGHTREC_REG_CLEAN:
235 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
236 break;
237 default:
238 break;
239 };
240 }
241}
242
98fa08a5 243static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
ba3814c1 244 jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
d16005f8 245{
98fa08a5 246 struct regcache *reg_cache = state->reg_cache;
d16005f8
PC		 247 struct native_register *regs_backup;
248 jit_state_t *_jit = block->_jit;
249 struct lightrec_branch *branch;
98fa08a5 250 const struct opcode *op = &block->opcode_list[offset],
cb72ea13 251 *ds = get_delay_slot(block->opcode_list, offset);
d16005f8 252 jit_node_t *addr;
cb72ea13 253 bool is_forward = (s16)op->i.imm >= 0;
684432ad 254 int op_cycles = lightrec_cycles_of_opcode(state->state, op->c);
03535202 255 u32 target_offset, cycles = state->cycles + op_cycles;
ba3814c1 256 bool no_indirection = false;
98fa08a5 257 u32 next_pc;
9259d748 258 u8 rs, rt;
d16005f8
PC		 259
260 jit_note(__FILE__, __LINE__);
261
03535202 262 if (!op_flag_no_ds(op->flags))
684432ad 263 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
d16005f8 264
03535202
PC		 265 state->cycles = -op_cycles;
266
267 if (!unconditional) {
268 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
269 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
270 _jit, op->i.rt, REG_EXT);
271
272 /* Unload dead registers before evaluating the branch */
273 if (OPT_EARLY_UNLOAD)
274 lightrec_do_early_unload(state, block, offset);
ba3814c1
PC		 275
276 if (op_flag_local_branch(op->flags) &&
cb72ea13 277 (op_flag_no_ds(op->flags) || !ds->opcode) &&
ba3814c1
PC		 278 is_forward && !lightrec_has_dirty_regs(reg_cache))
279 no_indirection = true;
280
281 if (no_indirection)
282 pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
03535202 283 }
d16005f8
PC		 284
285 if (cycles)
286 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
287
288 if (!unconditional) {
d16005f8 289 /* Generate the branch opcode */
ba3814c1
PC		 290 if (!no_indirection)
291 addr = jit_new_node_pww(code, NULL, rs, rt);
d16005f8
PC		 292
293 lightrec_free_regs(reg_cache);
294 regs_backup = lightrec_regcache_enter_branch(reg_cache);
295 }
296
03535202
PC		 297 if (op_flag_local_branch(op->flags)) {
298 /* Recompile the delay slot */
cb72ea13
PC		 299 if (!op_flag_no_ds(op->flags) && ds->opcode) {
300 /* Never handle load delays with local branches. */
301 state->no_load_delay = true;
03535202 302 lightrec_rec_opcode(state, block, offset + 1);
cb72ea13 303 }
d16005f8 304
9259d748
PC		 305 if (link)
306 update_ra_register(reg_cache, _jit, 31, block->pc, link);
d16005f8 307
03535202
PC		 308 /* Clean remaining registers */
309 lightrec_clean_regs(reg_cache, _jit);
d16005f8 310
98fa08a5 311 target_offset = offset + 1 + (s16)op->i.imm
03535202 312 - !!op_flag_no_ds(op->flags);
98fa08a5
PC		 313 pr_debug("Adding local branch to offset 0x%x\n",
314 target_offset << 2);
315 branch = &state->local_branches[
316 state->nb_local_branches++];
d16005f8 317
98fa08a5 318 branch->target = target_offset;
ba3814c1
PC		 319
320 if (no_indirection)
321 branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
322 else if (is_forward)
22eee2ac 323 branch->branch = jit_b();
d16005f8
PC		 324 else
325 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
326 }
327
03535202 328 if (!op_flag_local_branch(op->flags) || !is_forward) {
98fa08a5 329 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
cb72ea13 330 state->no_load_delay = op_flag_local_branch(op->flags);
98fa08a5 331 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
d16005f8
PC		 332 31, link, false);
333 }
334
335 if (!unconditional) {
ba3814c1
PC		 336 if (!no_indirection)
337 jit_patch(addr);
338
d16005f8
PC		 339 lightrec_regcache_leave_branch(reg_cache, regs_backup);
340
9259d748
PC		 341 if (bz && link)
342 update_ra_register(reg_cache, _jit, 31, block->pc, link);
d16005f8 343
cb72ea13
PC		 344 if (!op_flag_no_ds(op->flags) && ds->opcode) {
345 state->no_load_delay = true;
98fa08a5 346 lightrec_rec_opcode(state, block, offset + 1);
cb72ea13 347 }
d16005f8
PC		 348 }
349}
350
98fa08a5
PC		 351static void rec_BNE(struct lightrec_cstate *state,
352 const struct block *block, u16 offset)
d16005f8 353{
98fa08a5
PC		 354 union code c = block->opcode_list[offset].c;
355
d16005f8 356 _jit_name(block->_jit, __func__);
98fa08a5
PC		 357
358 if (c.i.rt == 0)
ba3814c1 359 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
98fa08a5 360 else
ba3814c1 361 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
d16005f8
PC		 362}
363
98fa08a5
PC		 364static void rec_BEQ(struct lightrec_cstate *state,
365 const struct block *block, u16 offset)
d16005f8 366{
98fa08a5
PC		 367 union code c = block->opcode_list[offset].c;
368
d16005f8 369 _jit_name(block->_jit, __func__);
98fa08a5
PC		 370
371 if (c.i.rt == 0)
ba3814c1 372 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
98fa08a5 373 else
ba3814c1 374 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
d16005f8
PC		 375}
376
98fa08a5
PC		 377static void rec_BLEZ(struct lightrec_cstate *state,
378 const struct block *block, u16 offset)
d16005f8 379{
98fa08a5
PC		 380 union code c = block->opcode_list[offset].c;
381
d16005f8 382 _jit_name(block->_jit, __func__);
ba3814c1 383 rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
d16005f8
PC		 384}
385
98fa08a5
PC		 386static void rec_BGTZ(struct lightrec_cstate *state,
387 const struct block *block, u16 offset)
d16005f8
PC		 388{
389 _jit_name(block->_jit, __func__);
ba3814c1 390 rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
d16005f8
PC		 391}
392
98fa08a5
PC		 393static void rec_regimm_BLTZ(struct lightrec_cstate *state,
394 const struct block *block, u16 offset)
d16005f8
PC		 395{
396 _jit_name(block->_jit, __func__);
ba3814c1 397 rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
d16005f8
PC		 398}
399
98fa08a5
PC		 400static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
401 const struct block *block, u16 offset)
d16005f8
PC		 402{
403 _jit_name(block->_jit, __func__);
ba3814c1 404 rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
98fa08a5 405 get_branch_pc(block, offset, 2), false, true);
d16005f8
PC		 406}
407
98fa08a5
PC		 408static void rec_regimm_BGEZ(struct lightrec_cstate *state,
409 const struct block *block, u16 offset)
d16005f8 410{
98fa08a5
PC		 411 union code c = block->opcode_list[offset].c;
412
d16005f8 413 _jit_name(block->_jit, __func__);
ba3814c1 414 rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
d16005f8
PC		 415}
416
98fa08a5
PC		 417static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
418 const struct block *block, u16 offset)
d16005f8 419{
98fa08a5 420 const struct opcode *op = &block->opcode_list[offset];
d16005f8 421 _jit_name(block->_jit, __func__);
ba3814c1 422 rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
98fa08a5
PC		 423 get_branch_pc(block, offset, 2),
424 !op->i.rs, true);
d16005f8
PC		 425}
426
684432ad
PC		 427static void rec_alloc_rs_rd(struct regcache *reg_cache,
428 jit_state_t *_jit,
429 const struct opcode *op,
430 u8 rs, u8 rd,
431 u8 in_flags, u8 out_flags,
432 u8 *rs_out, u8 *rd_out)
433{
434 bool unload, discard;
435 u32 unload_flags;
436
437 if (OPT_EARLY_UNLOAD) {
438 unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags);
439 unload = unload_flags == LIGHTREC_REG_UNLOAD;
440 discard = unload_flags == LIGHTREC_REG_DISCARD;
441 }
442
443 if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) {
444 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
445 lightrec_remap_reg(reg_cache, _jit, rs, rd, discard);
446 lightrec_set_reg_out_flags(reg_cache, rs, out_flags);
447 rd = rs;
448 } else {
449 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
450 rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags);
451 }
452
453 *rs_out = rs;
454 *rd_out = rd;
455}
456
98fa08a5
PC		 457static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
458 u16 offset, jit_code_t code, bool slti)
d16005f8 459{
98fa08a5
PC		 460 struct regcache *reg_cache = state->reg_cache;
461 union code c = block->opcode_list[offset].c;
d16005f8 462 jit_state_t *_jit = block->_jit;
98fa08a5
PC		 463 u8 rs, rt, out_flags = REG_EXT;
464
465 if (slti)
466 out_flags |= REG_ZEXT;
d16005f8
PC		 467
468 jit_note(__FILE__, __LINE__);
684432ad
PC		 469
470 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
471 c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt);
d16005f8 472
98fa08a5 473 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
d16005f8
PC		 474
475 lightrec_free_reg(reg_cache, rs);
476 lightrec_free_reg(reg_cache, rt);
477}
478
98fa08a5
PC		 479static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
480 u16 offset, jit_code_t code, bool out_ext)
d16005f8 481{
98fa08a5
PC		 482 struct regcache *reg_cache = state->reg_cache;
483 union code c = block->opcode_list[offset].c;
d16005f8
PC		 484 jit_state_t *_jit = block->_jit;
485 u8 rd, rt, rs;
486
487 jit_note(__FILE__, __LINE__);
684432ad 488
98fa08a5 489 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
684432ad
PC		 490 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
491 c.r.rs, c.r.rd, REG_EXT,
492 out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd);
d16005f8
PC		 493
494 jit_new_node_www(code, rd, rs, rt);
495
496 lightrec_free_reg(reg_cache, rs);
497 lightrec_free_reg(reg_cache, rt);
498 lightrec_free_reg(reg_cache, rd);
499}
500
98fa08a5
PC		 501static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
502 u16 offset, jit_code_t code)
d16005f8 503{
98fa08a5
PC		 504 struct regcache *reg_cache = state->reg_cache;
505 union code c = block->opcode_list[offset].c;
d16005f8 506 jit_state_t *_jit = block->_jit;
98fa08a5 507 u8 rd, rt, rs, temp, flags = 0;
d16005f8
PC		 508
509 jit_note(__FILE__, __LINE__);
d16005f8 510
98fa08a5
PC		 511 if (code == jit_code_rshr)
512 flags = REG_EXT;
513 else if (code == jit_code_rshr_u)
514 flags = REG_ZEXT;
d16005f8 515
684432ad
PC		 516 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
517 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
518 c.r.rt, c.r.rd, flags, flags, &rt, &rd);
d16005f8 519
684432ad 520 if (rt != rd) {
98fa08a5
PC		 521 jit_andi(rd, rs, 0x1f);
522 jit_new_node_www(code, rd, rt, rd);
523 } else {
524 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
525 jit_andi(temp, rs, 0x1f);
d16005f8 526 jit_new_node_www(code, rd, rt, temp);
98fa08a5
PC		 527 lightrec_free_reg(reg_cache, temp);
528 }
d16005f8
PC		 529
530 lightrec_free_reg(reg_cache, rs);
d16005f8
PC		 531 lightrec_free_reg(reg_cache, rt);
532 lightrec_free_reg(reg_cache, rd);
533}
534
02487de7
PC		 535static void rec_movi(struct lightrec_cstate *state,
536 const struct block *block, u16 offset)
537{
538 struct regcache *reg_cache = state->reg_cache;
539 union code c = block->opcode_list[offset].c;
540 jit_state_t *_jit = block->_jit;
541 u16 flags = REG_EXT;
684432ad 542 s32 value = (s32)(s16) c.i.imm;
02487de7
PC		 543 u8 rt;
544
684432ad
PC		 545 if (block->opcode_list[offset].flags & LIGHTREC_MOVI)
546 value += (s32)((u32)state->movi_temp[c.i.rt] << 16);
547
548 if (value >= 0)
02487de7
PC		 549 flags |= REG_ZEXT;
550
551 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
552
684432ad 553 jit_movi(rt, value);
02487de7
PC		 554
555 lightrec_free_reg(reg_cache, rt);
556}
557
98fa08a5
PC		 558static void rec_ADDIU(struct lightrec_cstate *state,
559 const struct block *block, u16 offset)
d16005f8 560{
684432ad
PC		 561 const struct opcode *op = &block->opcode_list[offset];
562
d16005f8 563 _jit_name(block->_jit, __func__);
02487de7 564
684432ad 565 if (op->i.rs && !(op->flags & LIGHTREC_MOVI))
02487de7
PC		 566 rec_alu_imm(state, block, offset, jit_code_addi, false);
567 else
568 rec_movi(state, block, offset);
d16005f8
PC		 569}
570
98fa08a5
PC		 571static void rec_ADDI(struct lightrec_cstate *state,
572 const struct block *block, u16 offset)
d16005f8
PC		 573{
574 /* TODO: Handle the exception? */
575 _jit_name(block->_jit, __func__);
02487de7 576 rec_ADDIU(state, block, offset);
d16005f8
PC		 577}
578
98fa08a5
PC		 579static void rec_SLTIU(struct lightrec_cstate *state,
580 const struct block *block, u16 offset)
d16005f8
PC		 581{
582 _jit_name(block->_jit, __func__);
98fa08a5 583 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
d16005f8
PC		 584}
585
98fa08a5
PC		 586static void rec_SLTI(struct lightrec_cstate *state,
587 const struct block *block, u16 offset)
d16005f8
PC		 588{
589 _jit_name(block->_jit, __func__);
98fa08a5 590 rec_alu_imm(state, block, offset, jit_code_lti, true);
d16005f8
PC		 591}
592
98fa08a5
PC		 593static void rec_ANDI(struct lightrec_cstate *state,
594 const struct block *block, u16 offset)
d16005f8 595{
98fa08a5
PC		 596 struct regcache *reg_cache = state->reg_cache;
597 union code c = block->opcode_list[offset].c;
d16005f8
PC		 598 jit_state_t *_jit = block->_jit;
599 u8 rs, rt;
600
601 _jit_name(block->_jit, __func__);
602 jit_note(__FILE__, __LINE__);
684432ad
PC		 603
604 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
605 c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt);
d16005f8
PC		 606
607 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
608 * casts to uint8_t / uint16_t. */
98fa08a5 609 if (c.i.imm == 0xff)
d16005f8 610 jit_extr_uc(rt, rs);
98fa08a5 611 else if (c.i.imm == 0xffff)
d16005f8
PC		 612 jit_extr_us(rt, rs);
613 else
98fa08a5
PC		 614 jit_andi(rt, rs, (u32)(u16) c.i.imm);
615
616 lightrec_free_reg(reg_cache, rs);
617 lightrec_free_reg(reg_cache, rt);
618}
619
620static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
621 u16 offset, jit_code_t code)
622{
623 struct regcache *reg_cache = state->reg_cache;
624 union code c = block->opcode_list[offset].c;
625 jit_state_t *_jit = block->_jit;
626 u8 rs, rt, flags;
627
628 jit_note(__FILE__, __LINE__);
684432ad
PC		 629
630 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
631 c.i.rs, c.i.rt, 0, 0, &rs, &rt);
98fa08a5
PC		 632
633 flags = lightrec_get_reg_in_flags(reg_cache, rs);
634 lightrec_set_reg_out_flags(reg_cache, rt, flags);
635
636 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
d16005f8
PC		 637
638 lightrec_free_reg(reg_cache, rs);
639 lightrec_free_reg(reg_cache, rt);
640}
641
98fa08a5
PC		 642
643static void rec_ORI(struct lightrec_cstate *state,
644 const struct block *block, u16 offset)
d16005f8 645{
684432ad
PC		 646 const struct opcode *op = &block->opcode_list[offset];
647 struct regcache *reg_cache = state->reg_cache;
648 jit_state_t *_jit = block->_jit;
649 s32 val;
650 u8 rt;
651
652 _jit_name(_jit, __func__);
653
654 if (op->flags & LIGHTREC_MOVI) {
655 rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT);
656
657 val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm;
658 jit_movi(rt, val);
659
660 lightrec_free_reg(reg_cache, rt);
661 } else {
662 rec_alu_or_xor(state, block, offset, jit_code_ori);
663 }
d16005f8
PC		 664}
665
98fa08a5
PC		 666static void rec_XORI(struct lightrec_cstate *state,
667 const struct block *block, u16 offset)
d16005f8
PC		 668{
669 _jit_name(block->_jit, __func__);
98fa08a5 670 rec_alu_or_xor(state, block, offset, jit_code_xori);
d16005f8
PC		 671}
672
98fa08a5
PC		 673static void rec_LUI(struct lightrec_cstate *state,
674 const struct block *block, u16 offset)
d16005f8 675{
98fa08a5
PC		 676 struct regcache *reg_cache = state->reg_cache;
677 union code c = block->opcode_list[offset].c;
d16005f8 678 jit_state_t *_jit = block->_jit;
98fa08a5 679 u8 rt, flags = REG_EXT;
d16005f8 680
684432ad
PC		 681 if (block->opcode_list[offset].flags & LIGHTREC_MOVI) {
682 state->movi_temp[c.i.rt] = c.i.imm;
683 return;
684 }
685
d16005f8
PC		 686 jit_name(__func__);
687 jit_note(__FILE__, __LINE__);
d16005f8 688
98fa08a5
PC		 689 if (!(c.i.imm & BIT(15)))
690 flags |= REG_ZEXT;
691
692 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
693
694 jit_movi(rt, (s32)(c.i.imm << 16));
d16005f8
PC		 695
696 lightrec_free_reg(reg_cache, rt);
697}
698
98fa08a5
PC		 699static void rec_special_ADDU(struct lightrec_cstate *state,
700 const struct block *block, u16 offset)
d16005f8
PC		 701{
702 _jit_name(block->_jit, __func__);
98fa08a5 703 rec_alu_special(state, block, offset, jit_code_addr, false);
d16005f8
PC		 704}
705
98fa08a5
PC		 706static void rec_special_ADD(struct lightrec_cstate *state,
707 const struct block *block, u16 offset)
d16005f8
PC		 708{
709 /* TODO: Handle the exception? */
710 _jit_name(block->_jit, __func__);
98fa08a5 711 rec_alu_special(state, block, offset, jit_code_addr, false);
d16005f8
PC		 712}
713
98fa08a5
PC		 714static void rec_special_SUBU(struct lightrec_cstate *state,
715 const struct block *block, u16 offset)
d16005f8
PC		 716{
717 _jit_name(block->_jit, __func__);
98fa08a5 718 rec_alu_special(state, block, offset, jit_code_subr, false);
d16005f8
PC		 719}
720
98fa08a5
PC		 721static void rec_special_SUB(struct lightrec_cstate *state,
722 const struct block *block, u16 offset)
d16005f8
PC		 723{
724 /* TODO: Handle the exception? */
725 _jit_name(block->_jit, __func__);
98fa08a5 726 rec_alu_special(state, block, offset, jit_code_subr, false);
d16005f8
PC		 727}
728
98fa08a5
PC		 729static void rec_special_AND(struct lightrec_cstate *state,
730 const struct block *block, u16 offset)
d16005f8 731{
98fa08a5
PC		 732 struct regcache *reg_cache = state->reg_cache;
733 union code c = block->opcode_list[offset].c;
734 jit_state_t *_jit = block->_jit;
735 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
736
d16005f8 737 _jit_name(block->_jit, __func__);
98fa08a5 738 jit_note(__FILE__, __LINE__);
684432ad 739
98fa08a5 740 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 741 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
742 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
98fa08a5
PC		 743
744 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
745 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
746
747 /* Z(rd) = Z(rs) | Z(rt) */
748 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
749
750 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
751 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
752 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
753 (REG_EXT & flags_rs & flags_rt))
754 flags_rd |= REG_EXT;
755
756 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
757
758 jit_andr(rd, rs, rt);
759
760 lightrec_free_reg(reg_cache, rs);
761 lightrec_free_reg(reg_cache, rt);
762 lightrec_free_reg(reg_cache, rd);
d16005f8
PC		 763}
764
98fa08a5
PC		 765static void rec_special_or_nor(struct lightrec_cstate *state,
766 const struct block *block, u16 offset, bool nor)
767{
768 struct regcache *reg_cache = state->reg_cache;
769 union code c = block->opcode_list[offset].c;
770 jit_state_t *_jit = block->_jit;
771 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
772
773 jit_note(__FILE__, __LINE__);
684432ad 774
98fa08a5 775 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 776 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
777 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
98fa08a5
PC		 778
779 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
780 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
781
782 /* or: Z(rd) = Z(rs) & Z(rt)
783 * nor: Z(rd) = 0 */
784 if (!nor)
785 flags_rd = REG_ZEXT & flags_rs & flags_rt;
786
6ce0b00a
PC		 787 /* E(rd) = E(rs) & E(rt) */
788 if (REG_EXT & flags_rs & flags_rt)
98fa08a5
PC		 789 flags_rd |= REG_EXT;
790
791 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
792
793 jit_orr(rd, rs, rt);
794
795 if (nor)
796 jit_comr(rd, rd);
797
798 lightrec_free_reg(reg_cache, rs);
799 lightrec_free_reg(reg_cache, rt);
800 lightrec_free_reg(reg_cache, rd);
801}
802
803static void rec_special_OR(struct lightrec_cstate *state,
804 const struct block *block, u16 offset)
d16005f8
PC		 805{
806 _jit_name(block->_jit, __func__);
98fa08a5 807 rec_special_or_nor(state, block, offset, false);
d16005f8
PC		 808}
809
98fa08a5
PC		 810static void rec_special_NOR(struct lightrec_cstate *state,
811 const struct block *block, u16 offset)
d16005f8
PC		 812{
813 _jit_name(block->_jit, __func__);
98fa08a5 814 rec_special_or_nor(state, block, offset, true);
d16005f8
PC		 815}
816
98fa08a5
PC		 817static void rec_special_XOR(struct lightrec_cstate *state,
818 const struct block *block, u16 offset)
d16005f8 819{
98fa08a5
PC		 820 struct regcache *reg_cache = state->reg_cache;
821 union code c = block->opcode_list[offset].c;
d16005f8 822 jit_state_t *_jit = block->_jit;
98fa08a5 823 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
d16005f8 824
98fa08a5
PC		 825 _jit_name(block->_jit, __func__);
826
827 jit_note(__FILE__, __LINE__);
684432ad 828
98fa08a5 829 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 830 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
831 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
d16005f8 832
98fa08a5
PC		 833 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
834 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
d16005f8 835
98fa08a5
PC		 836 /* Z(rd) = Z(rs) & Z(rt) */
837 flags_rd = REG_ZEXT & flags_rs & flags_rt;
838
839 /* E(rd) = E(rs) & E(rt) */
840 flags_rd |= REG_EXT & flags_rs & flags_rt;
841
842 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
843
844 jit_xorr(rd, rs, rt);
845
846 lightrec_free_reg(reg_cache, rs);
847 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 848 lightrec_free_reg(reg_cache, rd);
849}
850
98fa08a5
PC		 851static void rec_special_SLTU(struct lightrec_cstate *state,
852 const struct block *block, u16 offset)
d16005f8
PC		 853{
854 _jit_name(block->_jit, __func__);
98fa08a5 855 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
d16005f8
PC		 856}
857
98fa08a5
PC		 858static void rec_special_SLT(struct lightrec_cstate *state,
859 const struct block *block, u16 offset)
d16005f8
PC		 860{
861 _jit_name(block->_jit, __func__);
98fa08a5 862 rec_alu_special(state, block, offset, jit_code_ltr, true);
d16005f8
PC		 863}
864
98fa08a5
PC		 865static void rec_special_SLLV(struct lightrec_cstate *state,
866 const struct block *block, u16 offset)
d16005f8
PC		 867{
868 _jit_name(block->_jit, __func__);
98fa08a5 869 rec_alu_shiftv(state, block, offset, jit_code_lshr);
d16005f8
PC		 870}
871
98fa08a5
PC		 872static void rec_special_SRLV(struct lightrec_cstate *state,
873 const struct block *block, u16 offset)
d16005f8
PC		 874{
875 _jit_name(block->_jit, __func__);
98fa08a5 876 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
d16005f8
PC		 877}
878
98fa08a5
PC		 879static void rec_special_SRAV(struct lightrec_cstate *state,
880 const struct block *block, u16 offset)
d16005f8
PC		 881{
882 _jit_name(block->_jit, __func__);
98fa08a5 883 rec_alu_shiftv(state, block, offset, jit_code_rshr);
d16005f8
PC		 884}
885
98fa08a5
PC		 886static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
887 u16 offset, jit_code_t code)
d16005f8 888{
98fa08a5
PC		 889 struct regcache *reg_cache = state->reg_cache;
890 union code c = block->opcode_list[offset].c;
d16005f8 891 jit_state_t *_jit = block->_jit;
684432ad 892 u8 rd, rt, flags = 0, out_flags = 0;
d16005f8
PC		 893
894 jit_note(__FILE__, __LINE__);
895
98fa08a5
PC		 896 if (code == jit_code_rshi)
897 flags = REG_EXT;
898 else if (code == jit_code_rshi_u)
899 flags = REG_ZEXT;
d16005f8 900
98fa08a5
PC		 901 /* Input reg is zero-extended, if we SRL at least by one bit, we know
902 * the output reg will be both zero-extended and sign-extended. */
684432ad 903 out_flags = flags;
98fa08a5 904 if (code == jit_code_rshi_u && c.r.imm)
684432ad
PC		 905 out_flags |= REG_EXT;
906
907 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
908 c.r.rt, c.r.rd, flags, out_flags, &rt, &rd);
98fa08a5
PC		 909
910 jit_new_node_www(code, rd, rt, c.r.imm);
d16005f8
PC		 911
912 lightrec_free_reg(reg_cache, rt);
913 lightrec_free_reg(reg_cache, rd);
914}
915
98fa08a5
PC		 916static void rec_special_SLL(struct lightrec_cstate *state,
917 const struct block *block, u16 offset)
d16005f8
PC		 918{
919 _jit_name(block->_jit, __func__);
98fa08a5 920 rec_alu_shift(state, block, offset, jit_code_lshi);
d16005f8
PC		 921}
922
98fa08a5
PC		 923static void rec_special_SRL(struct lightrec_cstate *state,
924 const struct block *block, u16 offset)
d16005f8
PC		 925{
926 _jit_name(block->_jit, __func__);
98fa08a5 927 rec_alu_shift(state, block, offset, jit_code_rshi_u);
d16005f8
PC		 928}
929
98fa08a5
PC		 930static void rec_special_SRA(struct lightrec_cstate *state,
931 const struct block *block, u16 offset)
d16005f8
PC		 932{
933 _jit_name(block->_jit, __func__);
98fa08a5 934 rec_alu_shift(state, block, offset, jit_code_rshi);
d16005f8
PC		 935}
936
98fa08a5
PC		 937static void rec_alu_mult(struct lightrec_cstate *state,
938 const struct block *block, u16 offset, bool is_signed)
d16005f8 939{
98fa08a5
PC		 940 struct regcache *reg_cache = state->reg_cache;
941 union code c = block->opcode_list[offset].c;
03535202 942 u32 flags = block->opcode_list[offset].flags;
98fa08a5
PC		 943 u8 reg_lo = get_mult_div_lo(c);
944 u8 reg_hi = get_mult_div_hi(c);
d16005f8 945 jit_state_t *_jit = block->_jit;
98fa08a5 946 u8 lo, hi, rs, rt, rflags = 0;
5459088b
PC		 947 bool no_lo = op_flag_no_lo(flags);
948 bool no_hi = op_flag_no_hi(flags);
d16005f8
PC		 949
950 jit_note(__FILE__, __LINE__);
951
98fa08a5
PC		 952 if (is_signed)
953 rflags = REG_EXT;
954 else
955 rflags = REG_ZEXT;
956
957 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
958 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
959
5459088b 960 if (!no_lo)
98fa08a5 961 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
98fa08a5 962
5459088b 963 if (!no_hi)
98fa08a5
PC		 964 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
965
966 if (__WORDSIZE == 32) {
967 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
968 * operation if the MULT was detected a 32-bit only. */
5459088b 969 if (no_lo) {
98fa08a5 970 if (is_signed)
5459088b 971 jit_hmulr(hi, rs, rt);
98fa08a5 972 else
5459088b
PC		 973 jit_hmulr_u(hi, rs, rt);
974 } else if (no_hi) {
98fa08a5 975 jit_mulr(lo, rs, rt);
5459088b
PC		 976 } else if (is_signed) {
977 jit_qmulr(lo, hi, rs, rt);
978 } else {
979 jit_qmulr_u(lo, hi, rs, rt);
98fa08a5 980 }
d16005f8 981 } else {
98fa08a5 982 /* On 64-bit systems, do a 64*64->64 bit operation. */
5459088b 983 if (no_lo) {
98fa08a5
PC		 984 jit_mulr(hi, rs, rt);
985 jit_rshi(hi, hi, 32);
986 } else {
987 jit_mulr(lo, rs, rt);
d16005f8 988
98fa08a5 989 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
5459088b 990 if (!no_hi)
98fa08a5
PC		 991 jit_rshi(hi, lo, 32);
992 }
d16005f8
PC		 993 }
994
d16005f8
PC		 995 lightrec_free_reg(reg_cache, rs);
996 lightrec_free_reg(reg_cache, rt);
5459088b 997 if (!no_lo)
98fa08a5 998 lightrec_free_reg(reg_cache, lo);
5459088b 999 if (!no_hi)
d16005f8
PC		 1000 lightrec_free_reg(reg_cache, hi);
1001}
1002
98fa08a5
PC		 1003static void rec_alu_div(struct lightrec_cstate *state,
1004 const struct block *block, u16 offset, bool is_signed)
d16005f8 1005{
98fa08a5
PC		 1006 struct regcache *reg_cache = state->reg_cache;
1007 union code c = block->opcode_list[offset].c;
03535202
PC		 1008 u32 flags = block->opcode_list[offset].flags;
1009 bool no_check = op_flag_no_div_check(flags);
98fa08a5
PC		 1010 u8 reg_lo = get_mult_div_lo(c);
1011 u8 reg_hi = get_mult_div_hi(c);
d16005f8
PC		 1012 jit_state_t *_jit = block->_jit;
1013 jit_node_t *branch, *to_end;
fd58fa32 1014 u8 lo = 0, hi = 0, rs, rt, rflags = 0;
d16005f8
PC		 1015
1016 jit_note(__FILE__, __LINE__);
d16005f8 1017
98fa08a5
PC		 1018 if (is_signed)
1019 rflags = REG_EXT;
1020 else
1021 rflags = REG_ZEXT;
1022
1023 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
1024 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
1025
03535202 1026 if (!op_flag_no_lo(flags))
98fa08a5
PC		 1027 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
1028
03535202 1029 if (!op_flag_no_hi(flags))
98fa08a5 1030 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
d16005f8
PC		 1031
1032 /* Jump to special handler if dividing by zero */
98fa08a5
PC		 1033 if (!no_check)
1034 branch = jit_beqi(rt, 0);
d16005f8 1035
03535202 1036 if (op_flag_no_lo(flags)) {
98fa08a5
PC		 1037 if (is_signed)
1038 jit_remr(hi, rs, rt);
1039 else
1040 jit_remr_u(hi, rs, rt);
03535202 1041 } else if (op_flag_no_hi(flags)) {
98fa08a5
PC		 1042 if (is_signed)
1043 jit_divr(lo, rs, rt);
1044 else
1045 jit_divr_u(lo, rs, rt);
d16005f8 1046 } else {
98fa08a5
PC		 1047 if (is_signed)
1048 jit_qdivr(lo, hi, rs, rt);
1049 else
1050 jit_qdivr_u(lo, hi, rs, rt);
d16005f8 1051 }
d16005f8 1052
98fa08a5 1053 if (!no_check) {
98fa08a5 1054 /* Jump above the div-by-zero handler */
22eee2ac 1055 to_end = jit_b();
d16005f8 1056
98fa08a5
PC		 1057 jit_patch(branch);
1058
03535202 1059 if (!op_flag_no_lo(flags)) {
98fa08a5 1060 if (is_signed) {
9259d748 1061 jit_ltr(lo, rs, rt);
98fa08a5
PC		 1062 jit_lshi(lo, lo, 1);
1063 jit_subi(lo, lo, 1);
1064 } else {
9259d748 1065 jit_subi(lo, rt, 1);
98fa08a5
PC		 1066 }
1067 }
d16005f8 1068
03535202 1069 if (!op_flag_no_hi(flags))
98fa08a5 1070 jit_movr(hi, rs);
d16005f8 1071
98fa08a5
PC		 1072 jit_patch(to_end);
1073 }
d16005f8
PC		 1074
1075 lightrec_free_reg(reg_cache, rs);
1076 lightrec_free_reg(reg_cache, rt);
98fa08a5 1077
03535202 1078 if (!op_flag_no_lo(flags))
98fa08a5
PC		 1079 lightrec_free_reg(reg_cache, lo);
1080
03535202 1081 if (!op_flag_no_hi(flags))
98fa08a5 1082 lightrec_free_reg(reg_cache, hi);
d16005f8
PC		 1083}
1084
98fa08a5
PC		 1085static void rec_special_MULT(struct lightrec_cstate *state,
1086 const struct block *block, u16 offset)
d16005f8
PC		 1087{
1088 _jit_name(block->_jit, __func__);
98fa08a5 1089 rec_alu_mult(state, block, offset, true);
d16005f8
PC		 1090}
1091
98fa08a5
PC		 1092static void rec_special_MULTU(struct lightrec_cstate *state,
1093 const struct block *block, u16 offset)
d16005f8
PC		 1094{
1095 _jit_name(block->_jit, __func__);
98fa08a5 1096 rec_alu_mult(state, block, offset, false);
d16005f8
PC		 1097}
1098
98fa08a5
PC		 1099static void rec_special_DIV(struct lightrec_cstate *state,
1100 const struct block *block, u16 offset)
d16005f8
PC		 1101{
1102 _jit_name(block->_jit, __func__);
98fa08a5 1103 rec_alu_div(state, block, offset, true);
d16005f8
PC		 1104}
1105
98fa08a5
PC		 1106static void rec_special_DIVU(struct lightrec_cstate *state,
1107 const struct block *block, u16 offset)
d16005f8
PC		 1108{
1109 _jit_name(block->_jit, __func__);
98fa08a5 1110 rec_alu_div(state, block, offset, false);
d16005f8
PC		 1111}
1112
98fa08a5 1113static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
684432ad
PC		 1114 const struct block *block, u16 offset,
1115 u8 dst, u8 src)
d16005f8 1116{
98fa08a5 1117 struct regcache *reg_cache = state->reg_cache;
d16005f8
PC		 1118 jit_state_t *_jit = block->_jit;
1119
1120 jit_note(__FILE__, __LINE__);
684432ad
PC		 1121
1122 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
1123 src, dst, 0, REG_EXT, &src, &dst);
d16005f8 1124
d16005f8 1125 jit_extr_i(dst, src);
d16005f8
PC		 1126
1127 lightrec_free_reg(reg_cache, src);
1128 lightrec_free_reg(reg_cache, dst);
1129}
1130
98fa08a5
PC		 1131static void rec_special_MFHI(struct lightrec_cstate *state,
1132 const struct block *block, u16 offset)
d16005f8 1133{
98fa08a5
PC		 1134 union code c = block->opcode_list[offset].c;
1135
d16005f8 1136 _jit_name(block->_jit, __func__);
684432ad 1137 rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI);
d16005f8
PC		 1138}
1139
98fa08a5
PC		 1140static void rec_special_MTHI(struct lightrec_cstate *state,
1141 const struct block *block, u16 offset)
d16005f8 1142{
98fa08a5
PC		 1143 union code c = block->opcode_list[offset].c;
1144
d16005f8 1145 _jit_name(block->_jit, __func__);
684432ad 1146 rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs);
d16005f8
PC		 1147}
1148
98fa08a5
PC		 1149static void rec_special_MFLO(struct lightrec_cstate *state,
1150 const struct block *block, u16 offset)
d16005f8 1151{
98fa08a5
PC		 1152 union code c = block->opcode_list[offset].c;
1153
d16005f8 1154 _jit_name(block->_jit, __func__);
684432ad 1155 rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO);
d16005f8
PC		 1156}
1157
98fa08a5
PC		 1158static void rec_special_MTLO(struct lightrec_cstate *state,
1159 const struct block *block, u16 offset)
d16005f8 1160{
98fa08a5
PC		 1161 union code c = block->opcode_list[offset].c;
1162
d16005f8 1163 _jit_name(block->_jit, __func__);
684432ad 1164 rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs);
d16005f8
PC		 1165}
1166
ba3814c1
PC		 1167static void call_to_c_wrapper(struct lightrec_cstate *state,
1168 const struct block *block, u32 arg,
1169 enum c_wrappers wrapper)
d16005f8 1170{
98fa08a5 1171 struct regcache *reg_cache = state->reg_cache;
d16005f8 1172 jit_state_t *_jit = block->_jit;
ba3814c1 1173 s8 tmp, tmp2;
d16005f8 1174
ba3814c1
PC		 1175 /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1176 tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
d16005f8 1177
ba3814c1
PC		 1178 tmp = lightrec_get_reg_with_value(reg_cache,
1179 (intptr_t) state->state->wrappers_eps[wrapper]);
1180 if (tmp < 0) {
1181 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1182 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1183 offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1184
1185 lightrec_temp_set_value(reg_cache, tmp,
1186 (intptr_t) state->state->wrappers_eps[wrapper]);
22eee2ac
PC		 1187 }
1188
ba3814c1
PC		 1189 lightrec_free_reg(reg_cache, tmp2);
1190
1191#ifdef __mips__
1192 /* On MIPS, register t9 is always used as the target register for JALR.
1193 * Therefore if it does not contain the target address we must
1194 * invalidate it. */
1195 if (tmp != _T9)
1196 lightrec_unload_reg(reg_cache, _jit, _T9);
1197#endif
1198
1199 jit_prepare();
1200 jit_pushargi(arg);
1201
22eee2ac 1202 lightrec_regcache_mark_live(reg_cache, _jit);
98fa08a5 1203 jit_callr(tmp);
d16005f8 1204
98fa08a5 1205 lightrec_free_reg(reg_cache, tmp);
98fa08a5
PC		 1206 lightrec_regcache_mark_live(reg_cache, _jit);
1207}
1208
1209static void rec_io(struct lightrec_cstate *state,
1210 const struct block *block, u16 offset,
1211 bool load_rt, bool read_rt)
1212{
1213 struct regcache *reg_cache = state->reg_cache;
1214 jit_state_t *_jit = block->_jit;
1215 union code c = block->opcode_list[offset].c;
03535202 1216 u32 flags = block->opcode_list[offset].flags;
22eee2ac 1217 bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
98fa08a5 1218 u32 lut_entry;
cb72ea13 1219 u8 zero;
d16005f8 1220
98fa08a5 1221 jit_note(__FILE__, __LINE__);
d16005f8 1222
98fa08a5
PC		 1223 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1224
1225 if (read_rt && likely(c.i.rt))
1226 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
d16005f8 1227 else if (load_rt)
98fa08a5 1228 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
d16005f8 1229
cb72ea13
PC		 1230 if (op_flag_load_delay(flags) && !state->no_load_delay) {
1231 /* Clear state->in_delay_slot_n. This notifies the lightrec_rw
1232 * wrapper that it should write the REG_TEMP register instead of
1233 * the actual output register of the opcode. */
1234 zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1235 jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n),
1236 LIGHTREC_REG_STATE, zero);
1237 lightrec_free_reg(reg_cache, zero);
1238 }
1239
d16005f8 1240 if (is_tagged) {
ba3814c1 1241 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
d16005f8 1242 } else {
98fa08a5
PC		 1243 lut_entry = lightrec_get_lut_entry(block);
1244 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
ba3814c1 1245 C_WRAPPER_RW_GENERIC);
d16005f8 1246 }
d16005f8
PC		 1247}
1248
f5ee77ca 1249static u32 rec_ram_mask(const struct lightrec_state *state)
02487de7
PC		 1250{
1251 return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1252}
1253
13b02197
PC		 1254static u32 rec_io_mask(const struct lightrec_state *state)
1255{
1256 u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1257
2ec79b77 1258 return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0);
13b02197
PC		 1259}
1260
878e6cda
PC		 1261static void rec_add_offset(struct lightrec_cstate *cstate,
1262 jit_state_t *_jit, u8 reg_out, u8 reg_in,
1263 uintptr_t offset)
1264{
1265 struct regcache *reg_cache = cstate->reg_cache;
1266 u8 reg_imm;
1267
1268 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, offset);
1269 jit_addr(reg_out, reg_in, reg_imm);
1270
1271 lightrec_free_reg(reg_cache, reg_imm);
1272}
1273
1274static void rec_and_mask(struct lightrec_cstate *cstate,
1275 jit_state_t *_jit, u8 reg_out, u8 reg_in, u32 mask)
1276{
1277 struct regcache *reg_cache = cstate->reg_cache;
1278 u8 reg_imm;
1279
1280 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1281 jit_andr(reg_out, reg_in, reg_imm);
1282
1283 lightrec_free_reg(reg_cache, reg_imm);
1284}
1285
22eee2ac
PC		 1286static void rec_store_memory(struct lightrec_cstate *cstate,
1287 const struct block *block,
1288 u16 offset, jit_code_t code,
878e6cda
PC		 1289 jit_code_t swap_code, uintptr_t addr_offset,
1290 u32 addr_mask, bool invalidate)
22eee2ac 1291{
02487de7 1292 const struct lightrec_state *state = cstate->state;
22eee2ac
PC		 1293 struct regcache *reg_cache = cstate->reg_cache;
1294 struct opcode *op = &block->opcode_list[offset];
1295 jit_state_t *_jit = block->_jit;
1296 union code c = op->c;
f5ee77ca 1297 u8 rs, rt, tmp = 0, tmp2 = 0, tmp3, addr_reg, addr_reg2;
22eee2ac 1298 s16 imm = (s16)c.i.imm;
02487de7 1299 s32 simm = (s32)imm << (1 - lut_is_32bit(state));
22eee2ac 1300 s32 lut_offt = offsetof(struct lightrec_state, code_lut);
03535202 1301 bool no_mask = op_flag_no_mask(op->flags);
02487de7 1302 bool add_imm = c.i.imm &&
5459088b
PC		 1303 (c.i.op == OP_META_SWU
1304 || (!state->mirrors_mapped && !no_mask) || (invalidate &&
02487de7 1305 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
684432ad 1306 bool need_tmp = !no_mask || add_imm || invalidate;
9259d748 1307 bool swc2 = c.i.op == OP_SWC2;
cb72ea13 1308 u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
22eee2ac 1309
22eee2ac
PC		 1310 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1311 if (need_tmp)
1312 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1313
1314 addr_reg = rs;
1315
1316 if (add_imm) {
1317 jit_addi(tmp, addr_reg, (s16)c.i.imm);
0e720fb1 1318 lightrec_free_reg(reg_cache, rs);
22eee2ac
PC		 1319 addr_reg = tmp;
1320 imm = 0;
1321 } else if (simm) {
1322 lut_offt += simm;
1323 }
1324
1325 if (!no_mask) {
878e6cda 1326 rec_and_mask(cstate, _jit, tmp, addr_reg, addr_mask);
22eee2ac
PC		 1327 addr_reg = tmp;
1328 }
1329
22eee2ac 1330 if (addr_offset) {
ba3814c1 1331 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
878e6cda 1332 rec_add_offset(cstate, _jit, tmp2, addr_reg, addr_offset);
22eee2ac
PC		 1333 addr_reg2 = tmp2;
1334 } else {
1335 addr_reg2 = addr_reg;
1336 }
1337
684432ad
PC		 1338 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1339
9259d748 1340 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1341 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1342
1343 jit_new_node_ww(swap_code, tmp3, rt);
5459088b
PC		 1344
1345 if (c.i.op == OP_META_SWU)
1346 jit_unstr(addr_reg2, tmp3, LIGHTNING_UNALIGNED_32BIT);
1347 else
1348 jit_new_node_www(code, imm, addr_reg2, tmp3);
02487de7
PC		 1349
1350 lightrec_free_reg(reg_cache, tmp3);
5459088b
PC		 1351 } else if (c.i.op == OP_META_SWU) {
1352 jit_unstr(addr_reg2, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1353 } else {
1354 jit_new_node_www(code, imm, addr_reg2, rt);
1355 }
1356
22eee2ac
PC		 1357 lightrec_free_reg(reg_cache, rt);
1358
1359 if (invalidate) {
1360 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1361
1362 if (c.i.op != OP_SW) {
ba3814c1
PC		 1363 jit_andi(tmp, addr_reg, ~3);
1364 addr_reg = tmp;
22eee2ac
PC		 1365 }
1366
02487de7 1367 if (!lut_is_32bit(state)) {
ba3814c1
PC		 1368 jit_lshi(tmp, addr_reg, 1);
1369 addr_reg = tmp;
22eee2ac
PC		 1370 }
1371
02487de7
PC		 1372 if (addr_reg == rs && c.i.rs == 0) {
1373 addr_reg = LIGHTREC_REG_STATE;
1374 } else {
cb72ea13 1375 jit_add_state(tmp, addr_reg);
ba3814c1 1376 addr_reg = tmp;
22eee2ac
PC		 1377 }
1378
02487de7
PC		 1379 if (lut_is_32bit(state))
1380 jit_stxi_i(lut_offt, addr_reg, tmp3);
1381 else
1382 jit_stxi(lut_offt, addr_reg, tmp3);
22eee2ac
PC		 1383
1384 lightrec_free_reg(reg_cache, tmp3);
1385 }
1386
ba3814c1 1387 if (addr_offset)
22eee2ac
PC		 1388 lightrec_free_reg(reg_cache, tmp2);
1389 if (need_tmp)
1390 lightrec_free_reg(reg_cache, tmp);
1391 lightrec_free_reg(reg_cache, rs);
1392}
1393
1394static void rec_store_ram(struct lightrec_cstate *cstate,
1395 const struct block *block,
1396 u16 offset, jit_code_t code,
02487de7 1397 jit_code_t swap_code, bool invalidate)
22eee2ac 1398{
f5ee77ca 1399 const struct lightrec_state *state = cstate->state;
02487de7 1400
22eee2ac
PC		 1401 _jit_note(block->_jit, __FILE__, __LINE__);
1402
02487de7
PC		 1403 return rec_store_memory(cstate, block, offset, code, swap_code,
1404 state->offset_ram, rec_ram_mask(state),
1405 invalidate);
22eee2ac
PC		 1406}
1407
1408static void rec_store_scratch(struct lightrec_cstate *cstate,
02487de7
PC		 1409 const struct block *block, u16 offset,
1410 jit_code_t code, jit_code_t swap_code)
22eee2ac
PC		 1411{
1412 _jit_note(block->_jit, __FILE__, __LINE__);
1413
02487de7 1414 return rec_store_memory(cstate, block, offset, code, swap_code,
22eee2ac
PC		 1415 cstate->state->offset_scratch,
1416 0x1fffffff, false);
1417}
1418
ba3814c1
PC		 1419static void rec_store_io(struct lightrec_cstate *cstate,
1420 const struct block *block, u16 offset,
1421 jit_code_t code, jit_code_t swap_code)
1422{
1423 _jit_note(block->_jit, __FILE__, __LINE__);
1424
1425 return rec_store_memory(cstate, block, offset, code, swap_code,
1426 cstate->state->offset_io,
13b02197 1427 rec_io_mask(cstate->state), false);
ba3814c1
PC		 1428}
1429
98fa08a5
PC		 1430static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1431 const struct block *block,
02487de7
PC		 1432 u16 offset, jit_code_t code,
1433 jit_code_t swap_code)
d16005f8 1434{
f5ee77ca 1435 const struct lightrec_state *state = cstate->state;
878e6cda 1436 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
98fa08a5
PC		 1437 struct regcache *reg_cache = cstate->reg_cache;
1438 union code c = block->opcode_list[offset].c;
d16005f8
PC		 1439 jit_state_t *_jit = block->_jit;
1440 jit_node_t *to_not_ram, *to_end;
9259d748 1441 bool swc2 = c.i.op == OP_SWC2;
878e6cda 1442 u8 addr_reg, tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt;
d16005f8
PC		 1443 s16 imm;
1444
1445 jit_note(__FILE__, __LINE__);
98fa08a5 1446 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
d16005f8 1447 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5 1448
d16005f8 1449 /* Convert to KUNSEG and avoid RAM mirrors */
5459088b 1450 if ((c.i.op == OP_META_SWU || !state->mirrors_mapped) && c.i.imm) {
d16005f8 1451 imm = 0;
98fa08a5 1452 jit_addi(tmp, rs, (s16)c.i.imm);
878e6cda 1453 addr_reg = tmp;
d16005f8 1454 } else {
684432ad 1455 imm = (s16)c.i.imm;
878e6cda 1456 addr_reg = rs;
d16005f8
PC		 1457 }
1458
878e6cda
PC		 1459 rec_and_mask(cstate, _jit, tmp, addr_reg, 0x1f800000 | (ram_size - 1));
1460
d16005f8
PC		 1461 lightrec_free_reg(reg_cache, rs);
1462
1463 if (state->offset_ram != state->offset_scratch) {
684432ad
PC		 1464 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1465
d16005f8
PC		 1466 to_not_ram = jit_bmsi(tmp, BIT(28));
1467
1468 jit_movi(tmp2, state->offset_ram);
1469
22eee2ac 1470 to_end = jit_b();
d16005f8
PC		 1471 jit_patch(to_not_ram);
1472
1473 jit_movi(tmp2, state->offset_scratch);
1474 jit_patch(to_end);
1475 } else if (state->offset_ram) {
684432ad
PC		 1476 tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1477 state->offset_ram);
d16005f8
PC		 1478 }
1479
684432ad 1480 if (state->offset_ram || state->offset_scratch) {
d16005f8 1481 jit_addr(tmp, tmp, tmp2);
98fa08a5
PC		 1482 lightrec_free_reg(reg_cache, tmp2);
1483 }
d16005f8 1484
9259d748 1485 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
ba3814c1 1486
9259d748 1487 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1488 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1489
1490 jit_new_node_ww(swap_code, tmp2, rt);
5459088b
PC		 1491
1492 if (c.i.op == OP_META_SWU)
1493 jit_unstr(tmp, tmp2, LIGHTNING_UNALIGNED_32BIT);
1494 else
1495 jit_new_node_www(code, imm, tmp, tmp2);
02487de7
PC		 1496
1497 lightrec_free_reg(reg_cache, tmp2);
5459088b
PC		 1498 } else if (c.i.op == OP_META_SWU) {
1499 jit_unstr(tmp, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1500 } else {
1501 jit_new_node_www(code, imm, tmp, rt);
1502 }
d16005f8
PC		 1503
1504 lightrec_free_reg(reg_cache, rt);
1505 lightrec_free_reg(reg_cache, tmp);
1506}
1507
98fa08a5 1508static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
02487de7 1509 u16 offset, jit_code_t code, jit_code_t swap_code)
d16005f8 1510{
f5ee77ca 1511 const struct lightrec_state *state = cstate->state;
98fa08a5
PC		 1512 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1513 struct regcache *reg_cache = cstate->reg_cache;
1514 union code c = block->opcode_list[offset].c;
d16005f8 1515 jit_state_t *_jit = block->_jit;
98fa08a5 1516 jit_node_t *to_not_ram, *to_end;
9259d748 1517 bool swc2 = c.i.op == OP_SWC2;
878e6cda 1518 u8 addr_reg, tmp, tmp2, tmp3, rs, rt, reg_imm;
cb72ea13 1519 u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
878e6cda 1520 u32 mask;
f5ee77ca 1521 bool different_offsets = state->offset_ram != state->offset_scratch;
d16005f8
PC		 1522
1523 jit_note(__FILE__, __LINE__);
1524
98fa08a5 1525 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
d16005f8 1526 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5 1527 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
d16005f8
PC		 1528
1529 /* Convert to KUNSEG and avoid RAM mirrors */
98fa08a5
PC		 1530 if (c.i.imm) {
1531 jit_addi(tmp2, rs, (s16)c.i.imm);
878e6cda 1532 addr_reg = tmp2;
d16005f8 1533 } else {
878e6cda 1534 addr_reg = rs;
d16005f8
PC		 1535 }
1536
878e6cda
PC		 1537 rec_and_mask(cstate, _jit, tmp2, addr_reg, 0x1f800000 | (ram_size - 1));
1538
d16005f8
PC		 1539 lightrec_free_reg(reg_cache, rs);
1540 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1541
878e6cda
PC		 1542 mask = c.i.op == OP_SW ? RAM_SIZE - 1 : (RAM_SIZE - 1) & ~3;
1543 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1544
f5ee77ca 1545 if (different_offsets) {
9259d748 1546 to_not_ram = jit_bgti(tmp2, ram_size);
878e6cda 1547 addr_reg = tmp2;
9259d748
PC		 1548 } else {
1549 jit_lti_u(tmp, tmp2, ram_size);
1550 jit_movnr(tmp, tmp2, tmp);
878e6cda 1551 addr_reg = tmp;
9259d748 1552 }
98fa08a5 1553
d16005f8 1554 /* Compute the offset to the code LUT */
878e6cda 1555 jit_andr(tmp, addr_reg, reg_imm);
9259d748 1556
02487de7 1557 if (!lut_is_32bit(state))
98fa08a5 1558 jit_lshi(tmp, tmp, 1);
cb72ea13 1559 jit_add_state(tmp, tmp);
d16005f8
PC		 1560
1561 /* Write NULL to the code LUT to invalidate any block that's there */
02487de7
PC		 1562 if (lut_is_32bit(state))
1563 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1564 else
1565 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
d16005f8 1566
5459088b
PC		 1567 if (c.i.op == OP_META_SWU) {
1568 /* With a SWU opcode, we might have touched the following 32-bit
1569 * word, so invalidate it as well */
1570 if (lut_is_32bit(state)) {
1571 jit_stxi_i(offsetof(struct lightrec_state, code_lut) + 4,
1572 tmp, tmp3);
1573 } else {
1574 jit_stxi(offsetof(struct lightrec_state, code_lut)
1575 + sizeof(uintptr_t), tmp, tmp3);
1576 }
1577 }
1578
f5ee77ca 1579 if (different_offsets) {
d16005f8
PC		 1580 jit_movi(tmp, state->offset_ram);
1581
22eee2ac 1582 to_end = jit_b();
9259d748 1583 jit_patch(to_not_ram);
d16005f8
PC		 1584 }
1585
d16005f8
PC		 1586 if (state->offset_ram || state->offset_scratch)
1587 jit_movi(tmp, state->offset_scratch);
1588
f5ee77ca 1589 if (different_offsets)
d16005f8
PC		 1590 jit_patch(to_end);
1591
1592 if (state->offset_ram || state->offset_scratch)
1593 jit_addr(tmp2, tmp2, tmp);
1594
1595 lightrec_free_reg(reg_cache, tmp);
1596 lightrec_free_reg(reg_cache, tmp3);
878e6cda 1597 lightrec_free_reg(reg_cache, reg_imm);
d16005f8 1598
9259d748 1599 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
02487de7 1600
9259d748 1601 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1602 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1603
1604 jit_new_node_ww(swap_code, tmp, rt);
5459088b
PC		 1605
1606 if (c.i.op == OP_META_SWU)
1607 jit_unstr(tmp2, tmp, LIGHTNING_UNALIGNED_32BIT);
1608 else
1609 jit_new_node_www(code, 0, tmp2, tmp);
02487de7
PC		 1610
1611 lightrec_free_reg(reg_cache, tmp);
5459088b
PC		 1612 } else if (c.i.op == OP_META_SWU) {
1613 jit_unstr(tmp2, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1614 } else {
1615 jit_new_node_www(code, 0, tmp2, rt);
1616 }
d16005f8
PC		 1617
1618 lightrec_free_reg(reg_cache, rt);
1619 lightrec_free_reg(reg_cache, tmp2);
1620}
1621
98fa08a5 1622static void rec_store(struct lightrec_cstate *state,
02487de7
PC		 1623 const struct block *block, u16 offset,
1624 jit_code_t code, jit_code_t swap_code)
d16005f8 1625{
03535202 1626 u32 flags = block->opcode_list[offset].flags;
9259d748 1627 u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
03535202 1628 bool no_invalidate = op_flag_no_invalidate(flags) ||
684432ad 1629 (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY);
9259d748
PC		 1630 union code c = block->opcode_list[offset].c;
1631 bool is_swc2 = c.i.op == OP_SWC2;
1632
1633 if (is_swc2) {
1634 switch (mode) {
1635 case LIGHTREC_IO_RAM:
1636 case LIGHTREC_IO_SCRATCH:
1637 case LIGHTREC_IO_DIRECT:
1638 case LIGHTREC_IO_DIRECT_HW:
cb72ea13 1639 rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP);
9259d748
PC		 1640 break;
1641 default:
1642 break;
1643 }
1644 }
22eee2ac 1645
9259d748 1646 switch (mode) {
22eee2ac 1647 case LIGHTREC_IO_RAM:
02487de7
PC		 1648 rec_store_ram(state, block, offset, code,
1649 swap_code, !no_invalidate);
22eee2ac
PC		 1650 break;
1651 case LIGHTREC_IO_SCRATCH:
02487de7 1652 rec_store_scratch(state, block, offset, code, swap_code);
22eee2ac
PC		 1653 break;
1654 case LIGHTREC_IO_DIRECT:
02487de7
PC		 1655 if (no_invalidate) {
1656 rec_store_direct_no_invalidate(state, block, offset,
1657 code, swap_code);
1658 } else {
1659 rec_store_direct(state, block, offset, code, swap_code);
1660 }
22eee2ac 1661 break;
ba3814c1
PC		 1662 case LIGHTREC_IO_DIRECT_HW:
1663 rec_store_io(state, block, offset, code, swap_code);
1664 break;
22eee2ac 1665 default:
98fa08a5 1666 rec_io(state, block, offset, true, false);
9259d748 1667 return;
d16005f8 1668 }
9259d748
PC		 1669
1670 if (is_swc2)
cb72ea13 1671 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
d16005f8
PC		 1672}
1673
98fa08a5
PC		 1674static void rec_SB(struct lightrec_cstate *state,
1675 const struct block *block, u16 offset)
d16005f8
PC		 1676{
1677 _jit_name(block->_jit, __func__);
02487de7 1678 rec_store(state, block, offset, jit_code_stxi_c, 0);
d16005f8
PC		 1679}
1680
98fa08a5
PC		 1681static void rec_SH(struct lightrec_cstate *state,
1682 const struct block *block, u16 offset)
d16005f8
PC		 1683{
1684 _jit_name(block->_jit, __func__);
02487de7
PC		 1685 rec_store(state, block, offset,
1686 jit_code_stxi_s, jit_code_bswapr_us);
d16005f8
PC		 1687}
1688
98fa08a5
PC		 1689static void rec_SW(struct lightrec_cstate *state,
1690 const struct block *block, u16 offset)
1691
d16005f8 1692{
9259d748
PC		 1693 union code c = block->opcode_list[offset].c;
1694
1695 _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
02487de7
PC		 1696 rec_store(state, block, offset,
1697 jit_code_stxi_i, jit_code_bswapr_ui);
d16005f8
PC		 1698}
1699
98fa08a5
PC		 1700static void rec_SWL(struct lightrec_cstate *state,
1701 const struct block *block, u16 offset)
d16005f8
PC		 1702{
1703 _jit_name(block->_jit, __func__);
98fa08a5 1704 rec_io(state, block, offset, true, false);
d16005f8
PC		 1705}
1706
98fa08a5
PC		 1707static void rec_SWR(struct lightrec_cstate *state,
1708 const struct block *block, u16 offset)
d16005f8
PC		 1709{
1710 _jit_name(block->_jit, __func__);
98fa08a5 1711 rec_io(state, block, offset, true, false);
d16005f8
PC		 1712}
1713
22eee2ac 1714static void rec_load_memory(struct lightrec_cstate *cstate,
02487de7
PC		 1715 const struct block *block, u16 offset,
1716 jit_code_t code, jit_code_t swap_code, bool is_unsigned,
22eee2ac
PC		 1717 uintptr_t addr_offset, u32 addr_mask)
1718{
878e6cda 1719 struct lightrec_state *state = cstate->state;
22eee2ac
PC		 1720 struct regcache *reg_cache = cstate->reg_cache;
1721 struct opcode *op = &block->opcode_list[offset];
cb72ea13 1722 bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
22eee2ac 1723 jit_state_t *_jit = block->_jit;
9259d748 1724 u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
03535202 1725 bool no_mask = op_flag_no_mask(op->flags);
22eee2ac 1726 union code c = op->c;
02487de7 1727 s16 imm;
22eee2ac 1728
cb72ea13
PC		 1729 if (load_delay || c.i.op == OP_LWC2)
1730 out_reg = REG_TEMP;
9259d748
PC		 1731 else if (c.i.rt)
1732 out_reg = c.i.rt;
1733 else
22eee2ac
PC		 1734 return;
1735
1736 if (is_unsigned)
1737 flags |= REG_ZEXT;
1738
1739 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
9259d748 1740 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
22eee2ac 1741
5459088b 1742 if ((op->i.op == OP_META_LWU && c.i.imm)
878e6cda 1743 || (!state->mirrors_mapped && c.i.imm && !no_mask)) {
02487de7 1744 jit_addi(rt, rs, (s16)c.i.imm);
22eee2ac 1745 addr_reg = rt;
02487de7 1746 imm = 0;
22eee2ac
PC		 1747 } else {
1748 addr_reg = rs;
02487de7
PC		 1749 imm = (s16)c.i.imm;
1750 }
1751
5459088b
PC		 1752 if (op->i.op == OP_META_LWU)
1753 imm = LIGHTNING_UNALIGNED_32BIT;
1754
02487de7 1755 if (!no_mask) {
878e6cda 1756 rec_and_mask(cstate, _jit, rt, addr_reg, addr_mask);
02487de7 1757 addr_reg = rt;
22eee2ac
PC		 1758 }
1759
1760 if (addr_offset) {
878e6cda 1761 rec_add_offset(cstate, _jit, rt, addr_reg, addr_offset);
22eee2ac
PC		 1762 addr_reg = rt;
1763 }
1764
02487de7
PC		 1765 jit_new_node_www(code, rt, addr_reg, imm);
1766
1767 if (is_big_endian() && swap_code) {
1768 jit_new_node_ww(swap_code, rt, rt);
1769
1770 if (c.i.op == OP_LH)
1771 jit_extr_s(rt, rt);
1772 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1773 jit_extr_i(rt, rt);
1774 }
22eee2ac
PC		 1775
1776 lightrec_free_reg(reg_cache, rs);
1777 lightrec_free_reg(reg_cache, rt);
1778}
1779
1780static void rec_load_ram(struct lightrec_cstate *cstate,
02487de7
PC		 1781 const struct block *block, u16 offset,
1782 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1783{
1784 _jit_note(block->_jit, __FILE__, __LINE__);
1785
02487de7
PC		 1786 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1787 cstate->state->offset_ram, rec_ram_mask(cstate->state));
22eee2ac
PC		 1788}
1789
1790static void rec_load_bios(struct lightrec_cstate *cstate,
02487de7
PC		 1791 const struct block *block, u16 offset,
1792 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1793{
1794 _jit_note(block->_jit, __FILE__, __LINE__);
1795
02487de7 1796 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
22eee2ac
PC		 1797 cstate->state->offset_bios, 0x1fffffff);
1798}
1799
1800static void rec_load_scratch(struct lightrec_cstate *cstate,
02487de7
PC		 1801 const struct block *block, u16 offset,
1802 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1803{
1804 _jit_note(block->_jit, __FILE__, __LINE__);
1805
02487de7 1806 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
22eee2ac
PC		 1807 cstate->state->offset_scratch, 0x1fffffff);
1808}
1809
ba3814c1
PC		 1810static void rec_load_io(struct lightrec_cstate *cstate,
1811 const struct block *block, u16 offset,
1812 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1813{
1814 _jit_note(block->_jit, __FILE__, __LINE__);
1815
1816 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
13b02197 1817 cstate->state->offset_io, rec_io_mask(cstate->state));
ba3814c1
PC		 1818}
1819
02487de7
PC		 1820static void rec_load_direct(struct lightrec_cstate *cstate,
1821 const struct block *block, u16 offset,
1822 jit_code_t code, jit_code_t swap_code,
1823 bool is_unsigned)
d16005f8 1824{
f5ee77ca 1825 const struct lightrec_state *state = cstate->state;
98fa08a5 1826 struct regcache *reg_cache = cstate->reg_cache;
cb72ea13
PC		 1827 struct opcode *op = &block->opcode_list[offset];
1828 bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
d16005f8 1829 jit_state_t *_jit = block->_jit;
98fa08a5 1830 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
9259d748 1831 u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
f5ee77ca 1832 bool different_offsets = state->offset_bios != state->offset_scratch;
cb72ea13 1833 union code c = op->c;
684432ad
PC		 1834 s32 addr_mask;
1835 u32 reg_imm;
1836 s8 offt_reg;
d16005f8
PC		 1837 s16 imm;
1838
cb72ea13
PC		 1839 if (load_delay || c.i.op == OP_LWC2)
1840 out_reg = REG_TEMP;
9259d748
PC		 1841 else if (c.i.rt)
1842 out_reg = c.i.rt;
1843 else
d16005f8
PC		 1844 return;
1845
98fa08a5
PC		 1846 if (is_unsigned)
1847 flags |= REG_ZEXT;
1848
d16005f8 1849 jit_note(__FILE__, __LINE__);
98fa08a5 1850 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
9259d748 1851 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
d16005f8
PC		 1852
1853 if ((state->offset_ram == state->offset_bios &&
1854 state->offset_ram == state->offset_scratch &&
5459088b
PC		 1855 state->mirrors_mapped && c.i.op != OP_META_LWU)
1856 || !c.i.imm) {
d16005f8 1857 addr_reg = rs;
98fa08a5 1858 imm = (s16)c.i.imm;
d16005f8 1859 } else {
98fa08a5 1860 jit_addi(rt, rs, (s16)c.i.imm);
d16005f8
PC		 1861 addr_reg = rt;
1862 imm = 0;
1863
98fa08a5 1864 if (c.i.rs != c.i.rt)
d16005f8
PC		 1865 lightrec_free_reg(reg_cache, rs);
1866 }
1867
5459088b
PC		 1868 if (op->i.op == OP_META_LWU)
1869 imm = LIGHTNING_UNALIGNED_32BIT;
1870
d16005f8
PC		 1871 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1872
1873 if (state->offset_ram == state->offset_bios &&
1874 state->offset_ram == state->offset_scratch) {
684432ad
PC		 1875 if (!state->mirrors_mapped)
1876 addr_mask = 0x1f800000 | (RAM_SIZE - 1);
1877 else
1878 addr_mask = 0x1fffffff;
1879
1880 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1881 addr_mask);
d16005f8
PC		 1882 if (!state->mirrors_mapped) {
1883 jit_andi(tmp, addr_reg, BIT(28));
1884 jit_rshi_u(tmp, tmp, 28 - 22);
684432ad 1885 jit_orr(tmp, tmp, reg_imm);
d16005f8
PC		 1886 jit_andr(rt, addr_reg, tmp);
1887 } else {
684432ad 1888 jit_andr(rt, addr_reg, reg_imm);
d16005f8
PC		 1889 }
1890
684432ad
PC		 1891 lightrec_free_reg(reg_cache, reg_imm);
1892
1893 if (state->offset_ram) {
1894 offt_reg = lightrec_get_reg_with_value(reg_cache,
1895 state->offset_ram);
1896 if (offt_reg < 0) {
1897 jit_movi(tmp, state->offset_ram);
1898 lightrec_temp_set_value(reg_cache, tmp,
1899 state->offset_ram);
1900 } else {
1901 lightrec_free_reg(reg_cache, tmp);
1902 tmp = offt_reg;
1903 }
1904 }
d16005f8
PC		 1905 } else {
1906 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1907
1908 /* Convert to KUNSEG and avoid RAM mirrors */
1909 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1910
1911 if (state->offset_ram)
1912 jit_movi(tmp, state->offset_ram);
1913
22eee2ac 1914 to_end = jit_b();
d16005f8
PC		 1915
1916 jit_patch(to_not_ram);
1917
f5ee77ca 1918 if (different_offsets)
d16005f8
PC		 1919 to_not_bios = jit_bmci(addr_reg, BIT(22));
1920
1921 /* Convert to KUNSEG */
1922 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1923
1924 jit_movi(tmp, state->offset_bios);
1925
f5ee77ca 1926 if (different_offsets) {
22eee2ac 1927 to_end2 = jit_b();
d16005f8
PC		 1928
1929 jit_patch(to_not_bios);
1930
1931 /* Convert to KUNSEG */
1932 jit_andi(rt, addr_reg, 0x1f800fff);
1933
1934 if (state->offset_scratch)
1935 jit_movi(tmp, state->offset_scratch);
1936
1937 jit_patch(to_end2);
1938 }
1939
1940 jit_patch(to_end);
1941 }
1942
1943 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1944 jit_addr(rt, rt, tmp);
1945
1946 jit_new_node_www(code, rt, rt, imm);
1947
02487de7
PC		 1948 if (is_big_endian() && swap_code) {
1949 jit_new_node_ww(swap_code, rt, rt);
1950
1951 if (c.i.op == OP_LH)
1952 jit_extr_s(rt, rt);
1953 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1954 jit_extr_i(rt, rt);
1955 }
1956
d16005f8
PC		 1957 lightrec_free_reg(reg_cache, addr_reg);
1958 lightrec_free_reg(reg_cache, rt);
1959 lightrec_free_reg(reg_cache, tmp);
1960}
1961
98fa08a5 1962static void rec_load(struct lightrec_cstate *state, const struct block *block,
02487de7
PC		 1963 u16 offset, jit_code_t code, jit_code_t swap_code,
1964 bool is_unsigned)
d16005f8 1965{
9259d748
PC		 1966 const struct opcode *op = &block->opcode_list[offset];
1967 u32 flags = op->flags;
98fa08a5 1968
22eee2ac
PC		 1969 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1970 case LIGHTREC_IO_RAM:
02487de7 1971 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1972 break;
1973 case LIGHTREC_IO_BIOS:
02487de7 1974 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1975 break;
1976 case LIGHTREC_IO_SCRATCH:
02487de7 1977 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
22eee2ac 1978 break;
ba3814c1
PC		 1979 case LIGHTREC_IO_DIRECT_HW:
1980 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1981 break;
22eee2ac 1982 case LIGHTREC_IO_DIRECT:
02487de7 1983 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1984 break;
1985 default:
98fa08a5 1986 rec_io(state, block, offset, false, true);
9259d748
PC		 1987 return;
1988 }
1989
1990 if (op->i.op == OP_LWC2) {
cb72ea13
PC		 1991 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP);
1992 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
22eee2ac 1993 }
d16005f8
PC		 1994}
1995
98fa08a5 1996static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 1997{
1998 _jit_name(block->_jit, __func__);
02487de7 1999 rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
d16005f8
PC		 2000}
2001
98fa08a5 2002static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2003{
2004 _jit_name(block->_jit, __func__);
02487de7 2005 rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
d16005f8
PC		 2006}
2007
98fa08a5 2008static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2009{
ba3814c1
PC		 2010 jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
2011
d16005f8 2012 _jit_name(block->_jit, __func__);
ba3814c1 2013 rec_load(state, block, offset, code, jit_code_bswapr_us, false);
d16005f8
PC		 2014}
2015
98fa08a5 2016static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2017{
2018 _jit_name(block->_jit, __func__);
02487de7 2019 rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
d16005f8
PC		 2020}
2021
98fa08a5 2022static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2023{
2024 _jit_name(block->_jit, __func__);
98fa08a5 2025 rec_io(state, block, offset, true, true);
d16005f8
PC		 2026}
2027
98fa08a5 2028static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2029{
2030 _jit_name(block->_jit, __func__);
98fa08a5 2031 rec_io(state, block, offset, true, true);
d16005f8
PC		 2032}
2033
98fa08a5 2034static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2035{
9259d748 2036 union code c = block->opcode_list[offset].c;
ba3814c1
PC		 2037 jit_code_t code;
2038
2039 if (is_big_endian() && __WORDSIZE == 64)
2040 code = jit_code_ldxi_ui;
2041 else
2042 code = jit_code_ldxi_i;
2043
9259d748 2044 _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
ba3814c1 2045 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
d16005f8
PC		 2046}
2047
e26c79a8
PC		 2048static void rec_exit_early(struct lightrec_cstate *state,
2049 const struct block *block, u16 offset,
2050 u32 exit_code, u32 pc)
d16005f8 2051{
ba3814c1
PC		 2052 struct regcache *reg_cache = state->reg_cache;
2053 jit_state_t *_jit = block->_jit;
2054 u8 tmp;
2055
98fa08a5 2056 _jit_note(block->_jit, __FILE__, __LINE__);
d16005f8 2057
ba3814c1
PC		 2058 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2059
2060 jit_movi(tmp, exit_code);
2061 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2062 LIGHTREC_REG_STATE, tmp);
2063
cb72ea13
PC		 2064 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2065 offsetof(struct lightrec_state, target_cycle));
2066 jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE);
2067 jit_movi(LIGHTREC_REG_CYCLE, 0);
2068 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2069 LIGHTREC_REG_STATE, tmp);
2070 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2071 LIGHTREC_REG_STATE, tmp);
2072
ba3814c1 2073 lightrec_free_reg(reg_cache, tmp);
d16005f8 2074
e26c79a8 2075 lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true);
d16005f8
PC		 2076}
2077
98fa08a5
PC		 2078static void rec_special_SYSCALL(struct lightrec_cstate *state,
2079 const struct block *block, u16 offset)
d16005f8
PC		 2080{
2081 _jit_name(block->_jit, __func__);
e26c79a8
PC		 2082
2083 /* TODO: the return address should be "pc - 4" if we're a delay slot */
2084 rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL,
2085 get_ds_pc(block, offset, 0));
d16005f8
PC		 2086}
2087
98fa08a5
PC		 2088static void rec_special_BREAK(struct lightrec_cstate *state,
2089 const struct block *block, u16 offset)
d16005f8
PC		 2090{
2091 _jit_name(block->_jit, __func__);
e26c79a8
PC		 2092 rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK,
2093 get_ds_pc(block, offset, 0));
d16005f8
PC		 2094}
2095
fdf33147
PC		 2096static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2097{
2098 struct regcache *reg_cache = state->reg_cache;
2099 union code c = block->opcode_list[offset].c;
2100 jit_state_t *_jit = block->_jit;
2101
2102 jit_note(__FILE__, __LINE__);
9259d748
PC		 2103
2104 if (c.i.op != OP_SWC2)
2105 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
fdf33147
PC		 2106
2107 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
2108}
2109
fd58fa32 2110static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
98fa08a5
PC		 2111{
2112 struct regcache *reg_cache = state->reg_cache;
2113 union code c = block->opcode_list[offset].c;
2114 jit_state_t *_jit = block->_jit;
d16005f8 2115
98fa08a5
PC		 2116 jit_note(__FILE__, __LINE__);
2117 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
2118 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
cb72ea13 2119 lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false);
d16005f8 2120
ba3814c1 2121 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
d16005f8 2122
98fa08a5 2123 if (c.i.op == OP_CP0 &&
03535202 2124 !op_flag_no_ds(block->opcode_list[offset].flags) &&
98fa08a5
PC		 2125 (c.r.rd == 12 || c.r.rd == 13))
2126 lightrec_emit_end_of_block(state, block, offset, -1,
2127 get_ds_pc(block, offset, 1),
2128 0, 0, true);
d16005f8
PC		 2129}
2130
98fa08a5
PC		 2131static void
2132rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2133{
d16005f8 2134 struct regcache *reg_cache = state->reg_cache;
98fa08a5 2135 union code c = block->opcode_list[offset].c;
d16005f8 2136 jit_state_t *_jit = block->_jit;
98fa08a5 2137 u8 rt;
d16005f8
PC		 2138
2139 jit_note(__FILE__, __LINE__);
2140
98fa08a5 2141 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
d16005f8 2142
98fa08a5
PC		 2143 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
2144 offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
d16005f8 2145
98fa08a5
PC		 2146 lightrec_free_reg(reg_cache, rt);
2147}
d16005f8 2148
cb72ea13
PC		 2149static bool block_uses_icache(const struct lightrec_cstate *state,
2150 const struct block *block)
98fa08a5 2151{
cb72ea13 2152 const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM];
98fa08a5 2153 u32 pc = kunseg(block->pc);
d16005f8 2154
cb72ea13
PC		 2155 if (pc < map->pc || pc >= map->pc + map->length)
2156 return false;
2157
2158 return (block->pc >> 28) < 0xa;
d16005f8
PC		 2159}
2160
98fa08a5
PC		 2161static void
2162rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2163{
98fa08a5
PC		 2164 struct regcache *reg_cache = state->reg_cache;
2165 const union code c = block->opcode_list[offset].c;
2166 jit_state_t *_jit = block->_jit;
fd58fa32 2167 u8 rt, tmp = 0, tmp2, status;
0cf41071 2168 jit_node_t *to_end;
98fa08a5
PC		 2169
2170 jit_note(__FILE__, __LINE__);
2171
2172 switch(c.r.rd) {
2173 case 1:
2174 case 4:
2175 case 8:
2176 case 14:
2177 case 15:
2178 /* Those registers are read-only */
2179 return;
2180 default:
2181 break;
2182 }
2183
cb72ea13
PC		 2184 if (!block_uses_icache(state, block) && c.r.rd == 12) {
2185 /* If we are not running code from the RAM through kuseg or
2186 * kseg0, handle writes to the Status register in C; as the
2187 * code may toggle bit 16 which isolates the cache. Code
2188 * running from kuseg or kseg0 in RAM cannot do that. */
98fa08a5
PC		 2189 rec_mtc(state, block, offset);
2190 return;
2191 }
2192
2193 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
2194
2195 if (c.r.rd != 13) {
2196 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
2197 LIGHTREC_REG_STATE, rt);
2198 }
2199
2200 if (c.r.rd == 12 || c.r.rd == 13) {
2201 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2202 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2203 offsetof(struct lightrec_state, regs.cp0[13]));
fd58fa32
PC		 2204
2205 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5
PC		 2206 }
2207
2208 if (c.r.rd == 12) {
2209 status = rt;
2210 } else if (c.r.rd == 13) {
98fa08a5
PC		 2211 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
2212 jit_andi(tmp2, rt, 0x0300);
2213 jit_ori(tmp, tmp, 0x0300);
2214 jit_xori(tmp, tmp, 0x0300);
2215 jit_orr(tmp, tmp, tmp2);
2216 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2217 offsetof(struct lightrec_state, regs.cp0[12]));
2218 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
2219 LIGHTREC_REG_STATE, tmp);
2220 status = tmp2;
2221 }
2222
2223 if (c.r.rd == 12 || c.r.rd == 13) {
2224 /* Exit dynarec in case there's a software interrupt.
2225 * exit_flags = !!(status & tmp & 0x0300) & status; */
2226 jit_andr(tmp, tmp, status);
2227 jit_andi(tmp, tmp, 0x0300);
2228 jit_nei(tmp, tmp, 0);
2229 jit_andr(tmp, tmp, status);
fd58fa32
PC		 2230 }
2231
2232 if (c.r.rd == 12) {
2233 /* Exit dynarec in case we unmask a hardware interrupt.
2234 * exit_flags = !(~status & 0x401) */
2235
2236 jit_comr(tmp2, status);
2237 jit_andi(tmp2, tmp2, 0x401);
2238 jit_eqi(tmp2, tmp2, 0);
2239 jit_orr(tmp, tmp, tmp2);
2240 }
2241
0cf41071
PC		 2242 lightrec_free_reg(reg_cache, rt);
2243
fd58fa32 2244 if (c.r.rd == 12 || c.r.rd == 13) {
0cf41071 2245 to_end = jit_beqi(tmp, 0);
98fa08a5 2246
0cf41071
PC		 2247 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2248 offsetof(struct lightrec_state, target_cycle));
2249 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2250 jit_movi(LIGHTREC_REG_CYCLE, 0);
2251 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2252 LIGHTREC_REG_STATE, tmp2);
2253 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2254 LIGHTREC_REG_STATE, tmp2);
98fa08a5 2255
0cf41071
PC		 2256
2257 jit_patch(to_end);
2258 }
98fa08a5 2259
03535202 2260 if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
13b02197 2261 (c.r.rd == 12 || c.r.rd == 13)) {
684432ad 2262 state->cycles += lightrec_cycles_of_opcode(state->state, c);
13b02197
PC		 2263 lightrec_emit_eob(state, block, offset + 1);
2264 }
d16005f8
PC		 2265}
2266
98fa08a5
PC		 2267static void rec_cp0_MFC0(struct lightrec_cstate *state,
2268 const struct block *block, u16 offset)
d16005f8
PC		 2269{
2270 _jit_name(block->_jit, __func__);
98fa08a5 2271 rec_mfc0(state, block, offset);
d16005f8
PC		 2272}
2273
98fa08a5
PC		 2274static void rec_cp0_CFC0(struct lightrec_cstate *state,
2275 const struct block *block, u16 offset)
d16005f8
PC		 2276{
2277 _jit_name(block->_jit, __func__);
98fa08a5 2278 rec_mfc0(state, block, offset);
d16005f8
PC		 2279}
2280
98fa08a5
PC		 2281static void rec_cp0_MTC0(struct lightrec_cstate *state,
2282 const struct block *block, u16 offset)
d16005f8
PC		 2283{
2284 _jit_name(block->_jit, __func__);
98fa08a5 2285 rec_mtc0(state, block, offset);
d16005f8
PC		 2286}
2287
98fa08a5
PC		 2288static void rec_cp0_CTC0(struct lightrec_cstate *state,
2289 const struct block *block, u16 offset)
d16005f8
PC		 2290{
2291 _jit_name(block->_jit, __func__);
98fa08a5 2292 rec_mtc0(state, block, offset);
d16005f8
PC		 2293}
2294
02487de7
PC		 2295static unsigned int cp2d_i_offset(u8 reg)
2296{
2297 return offsetof(struct lightrec_state, regs.cp2d[reg]);
2298}
2299
2300static unsigned int cp2d_s_offset(u8 reg)
2301{
2302 return cp2d_i_offset(reg) + is_big_endian() * 2;
2303}
2304
2305static unsigned int cp2c_i_offset(u8 reg)
2306{
2307 return offsetof(struct lightrec_state, regs.cp2c[reg]);
2308}
2309
2310static unsigned int cp2c_s_offset(u8 reg)
2311{
2312 return cp2c_i_offset(reg) + is_big_endian() * 2;
2313}
2314
9259d748
PC		 2315static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2316 const struct block *block, u16 offset,
2317 u8 reg, u8 out_reg)
d16005f8 2318{
fd58fa32 2319 struct regcache *reg_cache = state->reg_cache;
fd58fa32
PC		 2320 jit_state_t *_jit = block->_jit;
2321 const u32 zext_regs = 0x300f0080;
2322 u8 rt, tmp, tmp2, tmp3, out, flags;
fd58fa32
PC		 2323 unsigned int i;
2324
d16005f8 2325 _jit_name(block->_jit, __func__);
fd58fa32 2326
fdf33147
PC		 2327 if (state->state->ops.cop2_notify) {
2328 /* We must call cop2_notify, handle that in C. */
2329 rec_mfc(state, block, offset);
2330 return;
2331 }
2332
fd58fa32 2333 flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
9259d748
PC		 2334 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2335
2336 if (reg == 15)
2337 reg = 14;
fd58fa32
PC		 2338
2339 switch (reg) {
2340 case 1:
2341 case 3:
2342 case 5:
2343 case 8:
2344 case 9:
2345 case 10:
2346 case 11:
02487de7 2347 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
fd58fa32
PC		 2348 break;
2349 case 7:
2350 case 16:
2351 case 17:
2352 case 18:
2353 case 19:
02487de7 2354 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
fd58fa32
PC		 2355 break;
2356 case 28:
2357 case 29:
2358 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2359 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2360 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2361
2362 for (i = 0; i < 3; i++) {
2363 out = i == 0 ? rt : tmp;
2364
02487de7 2365 jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
fd58fa32
PC		 2366 jit_movi(tmp2, 0x1f);
2367 jit_rshi(out, tmp, 7);
2368
2369 jit_ltr(tmp3, tmp2, out);
2370 jit_movnr(out, tmp2, tmp3);
2371
2372 jit_gei(tmp2, out, 0);
2373 jit_movzr(out, tmp2, tmp2);
2374
2375 if (i > 0) {
2376 jit_lshi(tmp, tmp, 5 * i);
2377 jit_orr(rt, rt, tmp);
2378 }
2379 }
2380
2381
2382 lightrec_free_reg(reg_cache, tmp);
2383 lightrec_free_reg(reg_cache, tmp2);
2384 lightrec_free_reg(reg_cache, tmp3);
2385 break;
2386 default:
02487de7 2387 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
fd58fa32
PC		 2388 break;
2389 }
2390
2391 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2392}
2393
9259d748
PC		 2394static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2395 const struct block *block, u16 offset)
2396{
2397 const union code c = block->opcode_list[offset].c;
2398
2399 rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2400}
2401
98fa08a5
PC		 2402static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2403 const struct block *block, u16 offset)
d16005f8 2404{
fd58fa32
PC		 2405 struct regcache *reg_cache = state->reg_cache;
2406 const union code c = block->opcode_list[offset].c;
2407 jit_state_t *_jit = block->_jit;
2408 u8 rt;
2409
d16005f8 2410 _jit_name(block->_jit, __func__);
fd58fa32 2411
fdf33147
PC		 2412 if (state->state->ops.cop2_notify) {
2413 /* We must call cop2_notify, handle that in C. */
2414 rec_mfc(state, block, offset);
2415 return;
2416 }
2417
fd58fa32
PC		 2418 switch (c.r.rd) {
2419 case 4:
2420 case 12:
2421 case 20:
2422 case 26:
2423 case 27:
2424 case 29:
2425 case 30:
2426 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
02487de7 2427 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
fd58fa32
PC		 2428 break;
2429 default:
2430 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
0cf41071 2431 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
fd58fa32
PC		 2432 break;
2433 }
2434
2435 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2436}
2437
9259d748
PC		 2438static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2439 const struct block *block, u16 offset,
2440 u8 reg, u8 in_reg)
d16005f8 2441{
fd58fa32 2442 struct regcache *reg_cache = state->reg_cache;
fd58fa32 2443 jit_state_t *_jit = block->_jit;
fd58fa32
PC		 2444 u8 rt, tmp, tmp2, flags = 0;
2445
d16005f8 2446 _jit_name(block->_jit, __func__);
fd58fa32 2447
fdf33147
PC		 2448 if (state->state->ops.cop2_notify) {
2449 /* We must call cop2_notify, handle that in C. */
2450 rec_mtc(state, block, offset);
2451 return;
2452 }
2453
9259d748 2454 if (reg == 31)
fd58fa32
PC		 2455 return;
2456
9259d748 2457 if (reg == 30)
fd58fa32
PC		 2458 flags |= REG_EXT;
2459
9259d748 2460 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
fd58fa32 2461
9259d748 2462 switch (reg) {
fd58fa32
PC		 2463 case 15:
2464 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
02487de7 2465 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
fd58fa32
PC		 2466
2467 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
02487de7 2468 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
fd58fa32 2469
02487de7
PC		 2470 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2471 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2472 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2473
2474 lightrec_free_reg(reg_cache, tmp);
2475 lightrec_free_reg(reg_cache, tmp2);
2476 break;
2477 case 28:
2478 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2479
2480 jit_lshi(tmp, rt, 7);
2481 jit_andi(tmp, tmp, 0xf80);
02487de7 2482 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2483
2484 jit_lshi(tmp, rt, 2);
2485 jit_andi(tmp, tmp, 0xf80);
02487de7 2486 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2487
2488 jit_rshi(tmp, rt, 3);
2489 jit_andi(tmp, tmp, 0xf80);
02487de7 2490 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2491
2492 lightrec_free_reg(reg_cache, tmp);
2493 break;
2494 case 30:
2495 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
fd58fa32
PC		 2496
2497 /* if (rt < 0) rt = ~rt; */
2498 jit_rshi(tmp, rt, 31);
2499 jit_xorr(tmp, rt, tmp);
2500
cb72ea13
PC		 2501 /* Count leading zeros */
2502 jit_clzr(tmp, tmp);
2503 if (__WORDSIZE != 32)
2504 jit_subi(tmp, tmp, __WORDSIZE - 32);
fd58fa32 2505
cb72ea13 2506 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2507
2508 lightrec_free_reg(reg_cache, tmp);
cb72ea13 2509 fallthrough;
fd58fa32 2510 default:
9259d748 2511 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2512 break;
2513 }
2514
2515 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2516}
2517
9259d748
PC		 2518static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2519 const struct block *block, u16 offset)
2520{
2521 const union code c = block->opcode_list[offset].c;
2522
2523 rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2524}
2525
98fa08a5
PC		 2526static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2527 const struct block *block, u16 offset)
d16005f8 2528{
fd58fa32
PC		 2529 struct regcache *reg_cache = state->reg_cache;
2530 const union code c = block->opcode_list[offset].c;
2531 jit_state_t *_jit = block->_jit;
2532 u8 rt, tmp, tmp2;
2533
98fa08a5 2534 _jit_name(block->_jit, __func__);
fd58fa32 2535
fdf33147
PC		 2536 if (state->state->ops.cop2_notify) {
2537 /* We must call cop2_notify, handle that in C. */
2538 rec_mtc(state, block, offset);
2539 return;
2540 }
2541
fd58fa32
PC		 2542 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2543
2544 switch (c.r.rd) {
2545 case 4:
2546 case 12:
2547 case 20:
2548 case 26:
2549 case 27:
2550 case 29:
2551 case 30:
02487de7 2552 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2553 break;
2554 case 31:
2555 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2556 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2557
2558 jit_andi(tmp, rt, 0x7f87e000);
2559 jit_nei(tmp, tmp, 0);
2560 jit_lshi(tmp, tmp, 31);
2561
2562 jit_andi(tmp2, rt, 0x7ffff000);
2563 jit_orr(tmp, tmp2, tmp);
2564
02487de7 2565 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2566
2567 lightrec_free_reg(reg_cache, tmp);
2568 lightrec_free_reg(reg_cache, tmp2);
2569 break;
2570
2571 default:
02487de7 2572 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2573 }
2574
2575 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2576}
2577
98fa08a5
PC		 2578static void rec_cp0_RFE(struct lightrec_cstate *state,
2579 const struct block *block, u16 offset)
d16005f8 2580{
98fa08a5 2581 struct regcache *reg_cache = state->reg_cache;
d16005f8 2582 jit_state_t *_jit = block->_jit;
98fa08a5 2583 u8 status, tmp;
d16005f8
PC		 2584
2585 jit_name(__func__);
2586 jit_note(__FILE__, __LINE__);
2587
98fa08a5
PC		 2588 status = lightrec_alloc_reg_temp(reg_cache, _jit);
2589 jit_ldxi_i(status, LIGHTREC_REG_STATE,
2590 offsetof(struct lightrec_state, regs.cp0[12]));
d16005f8 2591
98fa08a5 2592 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
d16005f8 2593
98fa08a5
PC		 2594 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2595 jit_rshi(tmp, status, 2);
2596 jit_andi(tmp, tmp, 0xf);
2597 jit_andi(status, status, ~0xful);
2598 jit_orr(status, status, tmp);
2599
2600 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2601 offsetof(struct lightrec_state, regs.cp0[13]));
2602 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2603 LIGHTREC_REG_STATE, status);
2604
2605 /* Exit dynarec in case there's a software interrupt.
2606 * exit_flags = !!(status & cause & 0x0300) & status; */
2607 jit_andr(tmp, tmp, status);
2608 jit_andi(tmp, tmp, 0x0300);
2609 jit_nei(tmp, tmp, 0);
2610 jit_andr(tmp, tmp, status);
2611 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2612 LIGHTREC_REG_STATE, tmp);
2613
2614 lightrec_free_reg(reg_cache, status);
d16005f8 2615 lightrec_free_reg(reg_cache, tmp);
d16005f8
PC		 2616}
2617
98fa08a5
PC		 2618static void rec_CP(struct lightrec_cstate *state,
2619 const struct block *block, u16 offset)
d16005f8 2620{
98fa08a5 2621 union code c = block->opcode_list[offset].c;
d16005f8
PC		 2622 jit_state_t *_jit = block->_jit;
2623
2624 jit_name(__func__);
2625 jit_note(__FILE__, __LINE__);
2626
ba3814c1 2627 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
d16005f8
PC		 2628}
2629
98fa08a5
PC		 2630static void rec_meta_MOV(struct lightrec_cstate *state,
2631 const struct block *block, u16 offset)
d16005f8 2632{
d16005f8 2633 struct regcache *reg_cache = state->reg_cache;
9259d748
PC		 2634 const struct opcode *op = &block->opcode_list[offset];
2635 union code c = op->c;
d16005f8 2636 jit_state_t *_jit = block->_jit;
9259d748 2637 bool unload_rd;
684432ad 2638 bool unload_rs, discard_rs;
d16005f8
PC		 2639 u8 rs, rd;
2640
2641 _jit_name(block->_jit, __func__);
2642 jit_note(__FILE__, __LINE__);
9259d748 2643
684432ad
PC		 2644 unload_rs = OPT_EARLY_UNLOAD
2645 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD;
2646 discard_rs = OPT_EARLY_UNLOAD
2647 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD;
2648
2649 if ((unload_rs || discard_rs) && c.m.rs) {
2650 /* If the source register is going to be unloaded or discarded,
2651 * then we can simply mark its host register as now pointing to
2652 * the destination register. */
2653 pr_debug("Remap %s to %s at offset 0x%x\n",
2654 lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd),
2655 offset << 2);
2656 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2657 lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs);
2658 lightrec_free_reg(reg_cache, rs);
2659 return;
2660 }
2661
9259d748
PC		 2662 unload_rd = OPT_EARLY_UNLOAD
2663 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2664
cb72ea13
PC		 2665 if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) {
2666 /* The source register is not yet loaded - we can load its value
2667 * from the register cache directly into the target register. */
2668 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2669
2670 jit_ldxi_i(rd, LIGHTREC_REG_STATE,
2671 offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2));
d16005f8 2672
cb72ea13
PC		 2673 lightrec_free_reg(reg_cache, rd);
2674 } else if (unload_rd) {
9259d748
PC		 2675 /* If the destination register will be unloaded right after the
2676 * MOV meta-opcode, we don't actually need to write any host
2677 * register - we can just store the source register directly to
2678 * the register cache, at the offset corresponding to the
2679 * destination register. */
cb72ea13
PC		 2680 lightrec_discard_reg_if_loaded(reg_cache, c.m.rd);
2681
2682 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
9259d748
PC		 2683
2684 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
cb72ea13 2685 + (c.m.rd << 2), LIGHTREC_REG_STATE, rs);
d16005f8 2686
98fa08a5 2687 lightrec_free_reg(reg_cache, rs);
9259d748 2688 } else {
cb72ea13
PC		 2689 if (c.m.rs)
2690 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2691
2692 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
9259d748 2693
cb72ea13 2694 if (c.m.rs == 0) {
9259d748 2695 jit_movi(rd, 0);
cb72ea13 2696 } else {
9259d748 2697 jit_extr_i(rd, rs);
cb72ea13
PC		 2698 lightrec_free_reg(reg_cache, rs);
2699 }
9259d748
PC		 2700
2701 lightrec_free_reg(reg_cache, rd);
2702 }
d16005f8
PC		 2703}
2704
98fa08a5
PC		 2705static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2706 const struct block *block,
2707 u16 offset)
d16005f8 2708{
98fa08a5
PC		 2709 struct regcache *reg_cache = state->reg_cache;
2710 union code c = block->opcode_list[offset].c;
d16005f8 2711 jit_state_t *_jit = block->_jit;
cb72ea13 2712 u8 rs, rd;
d16005f8 2713
98fa08a5 2714 _jit_name(block->_jit, __func__);
d16005f8
PC		 2715 jit_note(__FILE__, __LINE__);
2716
684432ad
PC		 2717 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2718 c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd);
d16005f8 2719
cb72ea13
PC		 2720 if (c.m.op == OP_META_EXTC)
2721 jit_extr_c(rd, rs);
98fa08a5 2722 else
cb72ea13 2723 jit_extr_s(rd, rs);
d16005f8 2724
98fa08a5 2725 lightrec_free_reg(reg_cache, rs);
cb72ea13 2726 lightrec_free_reg(reg_cache, rd);
d16005f8
PC		 2727}
2728
ba3814c1
PC		 2729static void rec_meta_MULT2(struct lightrec_cstate *state,
2730 const struct block *block,
2731 u16 offset)
2732{
2733 struct regcache *reg_cache = state->reg_cache;
2734 union code c = block->opcode_list[offset].c;
2735 jit_state_t *_jit = block->_jit;
2736 u8 reg_lo = get_mult_div_lo(c);
2737 u8 reg_hi = get_mult_div_hi(c);
2738 u32 flags = block->opcode_list[offset].flags;
2739 bool is_signed = c.i.op == OP_META_MULT2;
2740 u8 rs, lo, hi, rflags = 0, hiflags = 0;
0cf41071 2741 unsigned int i;
ba3814c1
PC		 2742
2743 if (!op_flag_no_hi(flags) && c.r.op < 32) {
2744 rflags = is_signed ? REG_EXT : REG_ZEXT;
2745 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2746 }
2747
2748 _jit_name(block->_jit, __func__);
2749 jit_note(__FILE__, __LINE__);
2750
2751 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2752
0cf41071
PC		 2753 /*
2754 * We must handle the case where one of the output registers is our rs
2755 * input register. Thanksfully, computing LO/HI can be done in any
2756 * order. Here, we make sure that the computation that overwrites the
2757 * input register is always performed last.
2758 */
2759 for (i = 0; i < 2; i++) {
2760 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2761 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2762
2763 if (c.r.op < 32)
2764 jit_lshi(lo, rs, c.r.op);
2765 else
2766 jit_movi(lo, 0);
ba3814c1 2767
0cf41071
PC		 2768 lightrec_free_reg(reg_cache, lo);
2769 continue;
2770 }
ba3814c1 2771
0cf41071
PC		 2772 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2773 hi = lightrec_alloc_reg_out(reg_cache, _jit,
2774 reg_hi, hiflags);
ba3814c1 2775
f5ee77ca 2776 if (c.r.op >= 32) {
0cf41071 2777 jit_lshi(hi, rs, c.r.op - 32);
f5ee77ca
PC		 2778 } else if (is_signed) {
2779 if (c.r.op)
2780 jit_rshi(hi, rs, 32 - c.r.op);
2781 else
2782 jit_rshi(hi, rs, 31);
2783 } else {
2784 if (c.r.op)
2785 jit_rshi_u(hi, rs, 32 - c.r.op);
2786 else
2787 jit_movi(hi, 0);
2788 }
ba3814c1 2789
0cf41071
PC		 2790 lightrec_free_reg(reg_cache, hi);
2791 }
ba3814c1
PC		 2792 }
2793
2794 lightrec_free_reg(reg_cache, rs);
2795
2796 _jit_name(block->_jit, __func__);
2797 jit_note(__FILE__, __LINE__);
2798}
2799
cb72ea13
PC		 2800static void rec_meta_COM(struct lightrec_cstate *state,
2801 const struct block *block, u16 offset)
2802{
2803 struct regcache *reg_cache = state->reg_cache;
2804 union code c = block->opcode_list[offset].c;
2805 jit_state_t *_jit = block->_jit;
2806 u8 rd, rs, flags;
2807
2808 jit_note(__FILE__, __LINE__);
684432ad
PC		 2809
2810 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2811 c.m.rs, c.m.rd, 0, 0, &rs, &rd);
cb72ea13
PC		 2812
2813 flags = lightrec_get_reg_in_flags(reg_cache, rs);
2814
2815 lightrec_set_reg_out_flags(reg_cache, rd,
2816 flags & REG_EXT);
2817
2818 jit_comr(rd, rs);
2819
2820 lightrec_free_reg(reg_cache, rs);
2821 lightrec_free_reg(reg_cache, rd);
2822}
2823
5459088b
PC		 2824static void rec_meta_LWU(struct lightrec_cstate *state,
2825 const struct block *block,
2826 u16 offset)
2827{
2828 jit_code_t code;
2829
2830 if (is_big_endian() && __WORDSIZE == 64)
2831 code = jit_code_unldr_u;
2832 else
2833 code = jit_code_unldr;
2834
2835 _jit_name(block->_jit, __func__);
2836 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2837}
2838
2839static void rec_meta_SWU(struct lightrec_cstate *state,
2840 const struct block *block,
2841 u16 offset)
2842{
2843 _jit_name(block->_jit, __func__);
2844 rec_store(state, block, offset, jit_code_unstr, jit_code_bswapr_ui);
2845}
2846
e26c79a8
PC		 2847static void unknown_opcode(struct lightrec_cstate *state,
2848 const struct block *block, u16 offset)
2849{
2850 rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP,
2851 block->pc + (offset << 2));
2852}
2853
d16005f8 2854static const lightrec_rec_func_t rec_standard[64] = {
98fa08a5 2855 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
d16005f8
PC		 2856 [OP_SPECIAL] = rec_SPECIAL,
2857 [OP_REGIMM] = rec_REGIMM,
2858 [OP_J] = rec_J,
2859 [OP_JAL] = rec_JAL,
2860 [OP_BEQ] = rec_BEQ,
2861 [OP_BNE] = rec_BNE,
2862 [OP_BLEZ] = rec_BLEZ,
2863 [OP_BGTZ] = rec_BGTZ,
2864 [OP_ADDI] = rec_ADDI,
2865 [OP_ADDIU] = rec_ADDIU,
2866 [OP_SLTI] = rec_SLTI,
2867 [OP_SLTIU] = rec_SLTIU,
2868 [OP_ANDI] = rec_ANDI,
2869 [OP_ORI] = rec_ORI,
2870 [OP_XORI] = rec_XORI,
2871 [OP_LUI] = rec_LUI,
2872 [OP_CP0] = rec_CP0,
2873 [OP_CP2] = rec_CP2,
2874 [OP_LB] = rec_LB,
2875 [OP_LH] = rec_LH,
2876 [OP_LWL] = rec_LWL,
2877 [OP_LW] = rec_LW,
2878 [OP_LBU] = rec_LBU,
2879 [OP_LHU] = rec_LHU,
2880 [OP_LWR] = rec_LWR,
2881 [OP_SB] = rec_SB,
2882 [OP_SH] = rec_SH,
2883 [OP_SWL] = rec_SWL,
2884 [OP_SW] = rec_SW,
2885 [OP_SWR] = rec_SWR,
9259d748
PC		 2886 [OP_LWC2] = rec_LW,
2887 [OP_SWC2] = rec_SW,
d16005f8 2888
cb72ea13 2889 [OP_META] = rec_META,
ba3814c1
PC		 2890 [OP_META_MULT2] = rec_meta_MULT2,
2891 [OP_META_MULTU2] = rec_meta_MULT2,
5459088b
PC		 2892 [OP_META_LWU] = rec_meta_LWU,
2893 [OP_META_SWU] = rec_meta_SWU,
d16005f8
PC		 2894};
2895
2896static const lightrec_rec_func_t rec_special[64] = {
98fa08a5 2897 SET_DEFAULT_ELM(rec_special, unknown_opcode),
d16005f8
PC		 2898 [OP_SPECIAL_SLL] = rec_special_SLL,
2899 [OP_SPECIAL_SRL] = rec_special_SRL,
2900 [OP_SPECIAL_SRA] = rec_special_SRA,
2901 [OP_SPECIAL_SLLV] = rec_special_SLLV,
2902 [OP_SPECIAL_SRLV] = rec_special_SRLV,
2903 [OP_SPECIAL_SRAV] = rec_special_SRAV,
2904 [OP_SPECIAL_JR] = rec_special_JR,
2905 [OP_SPECIAL_JALR] = rec_special_JALR,
2906 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
2907 [OP_SPECIAL_BREAK] = rec_special_BREAK,
2908 [OP_SPECIAL_MFHI] = rec_special_MFHI,
2909 [OP_SPECIAL_MTHI] = rec_special_MTHI,
2910 [OP_SPECIAL_MFLO] = rec_special_MFLO,
2911 [OP_SPECIAL_MTLO] = rec_special_MTLO,
2912 [OP_SPECIAL_MULT] = rec_special_MULT,
2913 [OP_SPECIAL_MULTU] = rec_special_MULTU,
2914 [OP_SPECIAL_DIV] = rec_special_DIV,
2915 [OP_SPECIAL_DIVU] = rec_special_DIVU,
2916 [OP_SPECIAL_ADD] = rec_special_ADD,
2917 [OP_SPECIAL_ADDU] = rec_special_ADDU,
2918 [OP_SPECIAL_SUB] = rec_special_SUB,
2919 [OP_SPECIAL_SUBU] = rec_special_SUBU,
2920 [OP_SPECIAL_AND] = rec_special_AND,
2921 [OP_SPECIAL_OR] = rec_special_OR,
2922 [OP_SPECIAL_XOR] = rec_special_XOR,
2923 [OP_SPECIAL_NOR] = rec_special_NOR,
2924 [OP_SPECIAL_SLT] = rec_special_SLT,
2925 [OP_SPECIAL_SLTU] = rec_special_SLTU,
2926};
2927
2928static const lightrec_rec_func_t rec_regimm[64] = {
98fa08a5 2929 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
d16005f8
PC		 2930 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
2931 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
2932 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
2933 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
2934};
2935
2936static const lightrec_rec_func_t rec_cp0[64] = {
98fa08a5 2937 SET_DEFAULT_ELM(rec_cp0, rec_CP),
d16005f8
PC		 2938 [OP_CP0_MFC0] = rec_cp0_MFC0,
2939 [OP_CP0_CFC0] = rec_cp0_CFC0,
2940 [OP_CP0_MTC0] = rec_cp0_MTC0,
2941 [OP_CP0_CTC0] = rec_cp0_CTC0,
2942 [OP_CP0_RFE] = rec_cp0_RFE,
2943};
2944
2945static const lightrec_rec_func_t rec_cp2_basic[64] = {
98fa08a5 2946 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
d16005f8
PC		 2947 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
2948 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
2949 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
2950 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
2951};
2952
cb72ea13
PC		 2953static const lightrec_rec_func_t rec_meta[64] = {
2954 SET_DEFAULT_ELM(rec_meta, unknown_opcode),
2955 [OP_META_MOV] = rec_meta_MOV,
2956 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
2957 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
2958 [OP_META_COM] = rec_meta_COM,
2959};
2960
98fa08a5
PC		 2961static void rec_SPECIAL(struct lightrec_cstate *state,
2962 const struct block *block, u16 offset)
d16005f8 2963{
98fa08a5
PC		 2964 union code c = block->opcode_list[offset].c;
2965 lightrec_rec_func_t f = rec_special[c.r.op];
2966
2967 if (!HAS_DEFAULT_ELM && unlikely(!f))
2968 unknown_opcode(state, block, offset);
d16005f8 2969 else
98fa08a5 2970 (*f)(state, block, offset);
d16005f8
PC		 2971}
2972
98fa08a5
PC		 2973static void rec_REGIMM(struct lightrec_cstate *state,
2974 const struct block *block, u16 offset)
d16005f8 2975{
98fa08a5
PC		 2976 union code c = block->opcode_list[offset].c;
2977 lightrec_rec_func_t f = rec_regimm[c.r.rt];
2978
2979 if (!HAS_DEFAULT_ELM && unlikely(!f))
2980 unknown_opcode(state, block, offset);
d16005f8 2981 else
98fa08a5 2982 (*f)(state, block, offset);
d16005f8
PC		 2983}
2984
98fa08a5
PC		 2985static void rec_CP0(struct lightrec_cstate *state,
2986 const struct block *block, u16 offset)
d16005f8 2987{
98fa08a5
PC		 2988 union code c = block->opcode_list[offset].c;
2989 lightrec_rec_func_t f = rec_cp0[c.r.rs];
2990
2991 if (!HAS_DEFAULT_ELM && unlikely(!f))
2992 rec_CP(state, block, offset);
d16005f8 2993 else
98fa08a5 2994 (*f)(state, block, offset);
d16005f8
PC		 2995}
2996
98fa08a5
PC		 2997static void rec_CP2(struct lightrec_cstate *state,
2998 const struct block *block, u16 offset)
d16005f8 2999{
98fa08a5
PC		 3000 union code c = block->opcode_list[offset].c;
3001
3002 if (c.r.op == OP_CP2_BASIC) {
3003 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
3004
3005 if (HAS_DEFAULT_ELM || likely(f)) {
3006 (*f)(state, block, offset);
d16005f8
PC		 3007 return;
3008 }
3009 }
3010
98fa08a5 3011 rec_CP(state, block, offset);
d16005f8
PC		 3012}
3013
cb72ea13
PC		 3014static void rec_META(struct lightrec_cstate *state,
3015 const struct block *block, u16 offset)
3016{
3017 union code c = block->opcode_list[offset].c;
3018 lightrec_rec_func_t f = rec_meta[c.m.op];
3019
3020 if (!HAS_DEFAULT_ELM && unlikely(!f))
3021 unknown_opcode(state, block, offset);
3022 else
3023 (*f)(state, block, offset);
3024}
3025
98fa08a5
PC		 3026void lightrec_rec_opcode(struct lightrec_cstate *state,
3027 const struct block *block, u16 offset)
d16005f8 3028{
98fa08a5
PC		 3029 struct regcache *reg_cache = state->reg_cache;
3030 struct lightrec_branch_target *target;
3031 const struct opcode *op = &block->opcode_list[offset];
3032 jit_state_t *_jit = block->_jit;
3033 lightrec_rec_func_t f;
03535202 3034 u16 unload_offset;
98fa08a5 3035
03535202
PC		 3036 if (op_flag_sync(op->flags)) {
3037 if (state->cycles)
3038 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
98fa08a5
PC		 3039 state->cycles = 0;
3040
3041 lightrec_storeback_regs(reg_cache, _jit);
3042 lightrec_regcache_reset(reg_cache);
3043
3044 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
3045 target = &state->targets[state->nb_targets++];
3046 target->offset = offset;
3047 target->label = jit_indirect();
3048 }
3049
3050 if (likely(op->opcode)) {
3051 f = rec_standard[op->i.op];
3052
3053 if (!HAS_DEFAULT_ELM && unlikely(!f))
3054 unknown_opcode(state, block, offset);
3055 else
3056 (*f)(state, block, offset);
3057 }
3058
03535202
PC		 3059 if (OPT_EARLY_UNLOAD) {
3060 unload_offset = offset +
3061 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
3062
3063 lightrec_do_early_unload(state, block, unload_offset);
98fa08a5 3064 }
cb72ea13
PC		 3065
3066 state->no_load_delay = false;
d16005f8 3067}
ARM optimized PCSX fork