notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
gpu_neon: revive the old tests
[pcsx_rearmed.git] / deps / lightrec / emitter.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC		 4 */
5
6#include "blockcache.h"
7#include "debug.h"
8#include "disassembler.h"
9#include "emitter.h"
98fa08a5 10#include "lightning-wrapper.h"
d16005f8
PC		 11#include "optimizer.h"
12#include "regcache.h"
13
d16005f8
PC		 14#include <stdbool.h>
15#include <stddef.h>
16
5459088b
PC		 17#define LIGHTNING_UNALIGNED_32BIT 4
18
98fa08a5 19typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
d16005f8
PC		 20
21/* Forward declarations */
98fa08a5
PC		 22static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
23static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
24static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
25static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
cb72ea13 26static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset);
9259d748
PC		 27static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
28 const struct block *block, u16 offset, u8 reg, u8 in_reg);
29static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
30 const struct block *block, u16 offset,
31 u8 reg, u8 out_reg);
d16005f8 32
ba3814c1 33static void
cb72ea13 34lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void))
ba3814c1
PC		 35{
36 /* Prevent jit_jmpi() from using our cycles register as a temporary */
37 jit_live(LIGHTREC_REG_CYCLE);
38
cb72ea13
PC		 39 jit_patch_abs(jit_jmpi(), fn);
40}
41
42static void
43lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
44{
45 lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func);
46}
47
48static void
49lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit)
50{
51 lightrec_jump_to_fn(_jit, state->state->ds_check_func);
ba3814c1
PC		 52}
53
9259d748
PC		 54static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
55 u8 ra_reg, u32 pc, u32 link)
56{
57 u8 link_reg;
58
59 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
60 lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
61 lightrec_free_reg(reg_cache, link_reg);
62}
63
98fa08a5
PC		 64static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
65 const struct block *block, u16 offset,
d16005f8
PC		 66 s8 reg_new_pc, u32 imm, u8 ra_reg,
67 u32 link, bool update_cycles)
68{
d16005f8 69 struct regcache *reg_cache = state->reg_cache;
d16005f8 70 jit_state_t *_jit = block->_jit;
98fa08a5 71 const struct opcode *op = &block->opcode_list[offset],
cb72ea13 72 *ds = get_delay_slot(block->opcode_list, offset);
684432ad 73 u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c);
d75d83f7 74 bool has_ds = has_delay_slot(op->c);
d16005f8
PC		 75
76 jit_note(__FILE__, __LINE__);
77
9259d748
PC		 78 if (link && ra_reg != reg_new_pc)
79 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
d16005f8 80
9259d748
PC		 81 if (reg_new_pc < 0)
82 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
83 else
84 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
d16005f8 85
9259d748
PC		 86 if (link && ra_reg == reg_new_pc) {
87 /* Handle the special case: JALR $r0, $r0
88 * In that case the target PC should be the old value of the
89 * register. */
90 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
d16005f8
PC		 91 }
92
d75d83f7 93 if (has_ds && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
684432ad 94 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
d16005f8
PC		 95
96 /* Recompile the delay slot */
cb72ea13 97 if (ds->c.opcode)
98fa08a5 98 lightrec_rec_opcode(state, block, offset + 1);
d16005f8
PC		 99 }
100
03535202
PC		 101 /* Clean the remaining registers */
102 lightrec_clean_regs(reg_cache, _jit);
d16005f8 103
d16005f8
PC		 104 if (cycles && update_cycles) {
105 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
106 pr_debug("EOB: %u cycles\n", cycles);
107 }
108
d75d83f7 109 if (has_ds && op_flag_load_delay(ds->flags)
a5a6f7b8 110 && opcode_has_load_delay(ds->c) && !state->no_load_delay) {
cb72ea13
PC		 111 /* If the delay slot is a load opcode, its target register
112 * will be written after the first opcode of the target is
113 * executed. Handle this by jumping to a special section of
114 * the dispatcher. It expects the loaded value to be in
115 * REG_TEMP, and the target register number to be in JIT_V1.*/
116 jit_movi(JIT_V1, ds->c.i.rt);
117
118 lightrec_jump_to_ds_check(state, _jit);
119 } else {
120 lightrec_jump_to_eob(state, _jit);
121 }
0e720fb1
PC		 122
123 lightrec_regcache_reset(reg_cache);
d16005f8
PC		 124}
125
cb72ea13
PC		 126void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state,
127 const struct block *block, u16 offset)
128{
129 struct regcache *reg_cache = state->reg_cache;
130 jit_state_t *_jit = block->_jit;
131
132 lightrec_clean_regs(reg_cache, _jit);
133
134 /* Call the interpreter with the block's address in JIT_V1 and the
135 * PC (which might have an offset) in JIT_V0. */
136 lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
137 block->pc + (offset << 2));
0e720fb1 138 if (lightrec_store_next_pc()) {
8afce295 139 jit_stxi_i(lightrec_offset(next_pc), LIGHTREC_REG_STATE, JIT_V0);
0e720fb1
PC		 140 }
141
cb72ea13
PC		 142 jit_movi(JIT_V1, (uintptr_t)block);
143
144 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
145 lightrec_jump_to_fn(_jit, state->state->interpreter_func);
146}
147
148static void lightrec_emit_eob(struct lightrec_cstate *state,
149 const struct block *block, u16 offset)
d16005f8 150{
d16005f8
PC		 151 struct regcache *reg_cache = state->reg_cache;
152 jit_state_t *_jit = block->_jit;
153
03535202 154 lightrec_clean_regs(reg_cache, _jit);
d16005f8 155
9259d748
PC		 156 lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
157 block->pc + (offset << 2));
0e720fb1 158 if (lightrec_store_next_pc()) {
8afce295 159 jit_stxi_i(lightrec_offset(next_pc), LIGHTREC_REG_STATE, JIT_V0);
0e720fb1
PC		 160 }
161
13b02197 162 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
d16005f8 163
ba3814c1 164 lightrec_jump_to_eob(state, _jit);
d16005f8
PC		 165}
166
98fa08a5 167static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 168{
9259d748 169 union code c = block->opcode_list[offset].c;
d16005f8
PC		 170
171 _jit_name(block->_jit, __func__);
9259d748 172 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
98fa08a5
PC		 173}
174
175static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
176{
98fa08a5
PC		 177 union code c = block->opcode_list[offset].c;
178
179 _jit_name(block->_jit, __func__);
9259d748 180 lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
98fa08a5 181 get_branch_pc(block, offset, 2), true);
d16005f8
PC		 182}
183
98fa08a5 184static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 185{
98fa08a5
PC		 186 union code c = block->opcode_list[offset].c;
187
d16005f8 188 _jit_name(block->_jit, __func__);
98fa08a5
PC		 189 lightrec_emit_end_of_block(state, block, offset, -1,
190 (block->pc & 0xf0000000) | (c.j.imm << 2),
191 31, 0, true);
d16005f8
PC		 192}
193
98fa08a5 194static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 195{
98fa08a5
PC		 196 union code c = block->opcode_list[offset].c;
197
d16005f8 198 _jit_name(block->_jit, __func__);
98fa08a5
PC		 199 lightrec_emit_end_of_block(state, block, offset, -1,
200 (block->pc & 0xf0000000) | (c.j.imm << 2),
201 31, get_branch_pc(block, offset, 2), true);
d16005f8
PC		 202}
203
03535202
PC		 204static void lightrec_do_early_unload(struct lightrec_cstate *state,
205 const struct block *block, u16 offset)
206{
207 struct regcache *reg_cache = state->reg_cache;
208 const struct opcode *op = &block->opcode_list[offset];
209 jit_state_t *_jit = block->_jit;
210 unsigned int i;
211 u8 reg;
212 struct {
213 u8 reg, op;
214 } reg_ops[3] = {
215 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
216 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
217 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
218 };
219
220 for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
221 reg = reg_ops[i].reg;
222
223 switch (reg_ops[i].op) {
224 case LIGHTREC_REG_UNLOAD:
225 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
226 break;
227
228 case LIGHTREC_REG_DISCARD:
229 lightrec_discard_reg_if_loaded(reg_cache, reg);
230 break;
231
232 case LIGHTREC_REG_CLEAN:
233 lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
234 break;
235 default:
236 break;
237 };
238 }
239}
240
98fa08a5 241static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
ba3814c1 242 jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
d16005f8 243{
98fa08a5 244 struct regcache *reg_cache = state->reg_cache;
d16005f8
PC		 245 struct native_register *regs_backup;
246 jit_state_t *_jit = block->_jit;
247 struct lightrec_branch *branch;
98fa08a5 248 const struct opcode *op = &block->opcode_list[offset],
cb72ea13 249 *ds = get_delay_slot(block->opcode_list, offset);
d16005f8 250 jit_node_t *addr;
cb72ea13 251 bool is_forward = (s16)op->i.imm >= 0;
684432ad 252 int op_cycles = lightrec_cycles_of_opcode(state->state, op->c);
03535202 253 u32 target_offset, cycles = state->cycles + op_cycles;
ba3814c1 254 bool no_indirection = false;
98fa08a5 255 u32 next_pc;
9259d748 256 u8 rs, rt;
d16005f8
PC		 257
258 jit_note(__FILE__, __LINE__);
259
03535202 260 if (!op_flag_no_ds(op->flags))
684432ad 261 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
d16005f8 262
03535202
PC		 263 state->cycles = -op_cycles;
264
265 if (!unconditional) {
266 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
267 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
268 _jit, op->i.rt, REG_EXT);
269
270 /* Unload dead registers before evaluating the branch */
271 if (OPT_EARLY_UNLOAD)
272 lightrec_do_early_unload(state, block, offset);
ba3814c1
PC		 273
274 if (op_flag_local_branch(op->flags) &&
cb72ea13 275 (op_flag_no_ds(op->flags) || !ds->opcode) &&
ba3814c1
PC		 276 is_forward && !lightrec_has_dirty_regs(reg_cache))
277 no_indirection = true;
278
279 if (no_indirection)
280 pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
03535202 281 }
d16005f8
PC		 282
283 if (cycles)
284 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
285
286 if (!unconditional) {
d16005f8 287 /* Generate the branch opcode */
ba3814c1
PC		 288 if (!no_indirection)
289 addr = jit_new_node_pww(code, NULL, rs, rt);
d16005f8
PC		 290
291 lightrec_free_regs(reg_cache);
292 regs_backup = lightrec_regcache_enter_branch(reg_cache);
293 }
294
03535202
PC		 295 if (op_flag_local_branch(op->flags)) {
296 /* Recompile the delay slot */
cb72ea13
PC		 297 if (!op_flag_no_ds(op->flags) && ds->opcode) {
298 /* Never handle load delays with local branches. */
299 state->no_load_delay = true;
03535202 300 lightrec_rec_opcode(state, block, offset + 1);
cb72ea13 301 }
d16005f8 302
9259d748
PC		 303 if (link)
304 update_ra_register(reg_cache, _jit, 31, block->pc, link);
d16005f8 305
03535202
PC		 306 /* Clean remaining registers */
307 lightrec_clean_regs(reg_cache, _jit);
d16005f8 308
98fa08a5 309 target_offset = offset + 1 + (s16)op->i.imm
03535202 310 - !!op_flag_no_ds(op->flags);
98fa08a5
PC		 311 pr_debug("Adding local branch to offset 0x%x\n",
312 target_offset << 2);
313 branch = &state->local_branches[
314 state->nb_local_branches++];
d16005f8 315
98fa08a5 316 branch->target = target_offset;
ba3814c1
PC		 317
318 if (no_indirection)
319 branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
320 else if (is_forward)
22eee2ac 321 branch->branch = jit_b();
d16005f8
PC		 322 else
323 branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
324 }
325
03535202 326 if (!op_flag_local_branch(op->flags) || !is_forward) {
98fa08a5 327 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
cb72ea13 328 state->no_load_delay = op_flag_local_branch(op->flags);
98fa08a5 329 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
d16005f8
PC		 330 31, link, false);
331 }
332
333 if (!unconditional) {
ba3814c1
PC		 334 if (!no_indirection)
335 jit_patch(addr);
336
d16005f8
PC		 337 lightrec_regcache_leave_branch(reg_cache, regs_backup);
338
9259d748
PC		 339 if (bz && link)
340 update_ra_register(reg_cache, _jit, 31, block->pc, link);
d16005f8 341
cb72ea13
PC		 342 if (!op_flag_no_ds(op->flags) && ds->opcode) {
343 state->no_load_delay = true;
98fa08a5 344 lightrec_rec_opcode(state, block, offset + 1);
cb72ea13 345 }
d16005f8
PC		 346 }
347}
348
98fa08a5
PC		 349static void rec_BNE(struct lightrec_cstate *state,
350 const struct block *block, u16 offset)
d16005f8 351{
98fa08a5
PC		 352 union code c = block->opcode_list[offset].c;
353
d16005f8 354 _jit_name(block->_jit, __func__);
98fa08a5
PC		 355
356 if (c.i.rt == 0)
ba3814c1 357 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
98fa08a5 358 else
ba3814c1 359 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
d16005f8
PC		 360}
361
98fa08a5
PC		 362static void rec_BEQ(struct lightrec_cstate *state,
363 const struct block *block, u16 offset)
d16005f8 364{
98fa08a5
PC		 365 union code c = block->opcode_list[offset].c;
366
d16005f8 367 _jit_name(block->_jit, __func__);
98fa08a5
PC		 368
369 if (c.i.rt == 0)
ba3814c1 370 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
98fa08a5 371 else
ba3814c1 372 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
d16005f8
PC		 373}
374
98fa08a5
PC		 375static void rec_BLEZ(struct lightrec_cstate *state,
376 const struct block *block, u16 offset)
d16005f8 377{
98fa08a5
PC		 378 union code c = block->opcode_list[offset].c;
379
d16005f8 380 _jit_name(block->_jit, __func__);
ba3814c1 381 rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
d16005f8
PC		 382}
383
98fa08a5
PC		 384static void rec_BGTZ(struct lightrec_cstate *state,
385 const struct block *block, u16 offset)
d16005f8
PC		 386{
387 _jit_name(block->_jit, __func__);
ba3814c1 388 rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
d16005f8
PC		 389}
390
98fa08a5
PC		 391static void rec_regimm_BLTZ(struct lightrec_cstate *state,
392 const struct block *block, u16 offset)
d16005f8
PC		 393{
394 _jit_name(block->_jit, __func__);
ba3814c1 395 rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
d16005f8
PC		 396}
397
98fa08a5
PC		 398static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
399 const struct block *block, u16 offset)
d16005f8
PC		 400{
401 _jit_name(block->_jit, __func__);
ba3814c1 402 rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
98fa08a5 403 get_branch_pc(block, offset, 2), false, true);
d16005f8
PC		 404}
405
98fa08a5
PC		 406static void rec_regimm_BGEZ(struct lightrec_cstate *state,
407 const struct block *block, u16 offset)
d16005f8 408{
98fa08a5
PC		 409 union code c = block->opcode_list[offset].c;
410
d16005f8 411 _jit_name(block->_jit, __func__);
ba3814c1 412 rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
d16005f8
PC		 413}
414
98fa08a5
PC		 415static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
416 const struct block *block, u16 offset)
d16005f8 417{
98fa08a5 418 const struct opcode *op = &block->opcode_list[offset];
d16005f8 419 _jit_name(block->_jit, __func__);
ba3814c1 420 rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
98fa08a5
PC		 421 get_branch_pc(block, offset, 2),
422 !op->i.rs, true);
d16005f8
PC		 423}
424
684432ad
PC		 425static void rec_alloc_rs_rd(struct regcache *reg_cache,
426 jit_state_t *_jit,
427 const struct opcode *op,
428 u8 rs, u8 rd,
429 u8 in_flags, u8 out_flags,
430 u8 *rs_out, u8 *rd_out)
431{
432 bool unload, discard;
433 u32 unload_flags;
434
435 if (OPT_EARLY_UNLOAD) {
436 unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags);
437 unload = unload_flags == LIGHTREC_REG_UNLOAD;
438 discard = unload_flags == LIGHTREC_REG_DISCARD;
439 }
440
441 if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) {
442 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
443 lightrec_remap_reg(reg_cache, _jit, rs, rd, discard);
444 lightrec_set_reg_out_flags(reg_cache, rs, out_flags);
445 rd = rs;
446 } else {
447 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
448 rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags);
449 }
450
451 *rs_out = rs;
452 *rd_out = rd;
453}
454
98fa08a5
PC		 455static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
456 u16 offset, jit_code_t code, bool slti)
d16005f8 457{
98fa08a5
PC		 458 struct regcache *reg_cache = state->reg_cache;
459 union code c = block->opcode_list[offset].c;
d16005f8 460 jit_state_t *_jit = block->_jit;
98fa08a5
PC		 461 u8 rs, rt, out_flags = REG_EXT;
462
463 if (slti)
464 out_flags |= REG_ZEXT;
d16005f8
PC		 465
466 jit_note(__FILE__, __LINE__);
684432ad
PC		 467
468 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
469 c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt);
d16005f8 470
98fa08a5 471 jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
d16005f8
PC		 472
473 lightrec_free_reg(reg_cache, rs);
474 lightrec_free_reg(reg_cache, rt);
475}
476
98fa08a5
PC		 477static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
478 u16 offset, jit_code_t code, bool out_ext)
d16005f8 479{
98fa08a5
PC		 480 struct regcache *reg_cache = state->reg_cache;
481 union code c = block->opcode_list[offset].c;
d16005f8
PC		 482 jit_state_t *_jit = block->_jit;
483 u8 rd, rt, rs;
484
485 jit_note(__FILE__, __LINE__);
684432ad 486
98fa08a5 487 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
684432ad
PC		 488 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
489 c.r.rs, c.r.rd, REG_EXT,
490 out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd);
d16005f8
PC		 491
492 jit_new_node_www(code, rd, rs, rt);
493
494 lightrec_free_reg(reg_cache, rs);
495 lightrec_free_reg(reg_cache, rt);
496 lightrec_free_reg(reg_cache, rd);
497}
498
98fa08a5
PC		 499static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
500 u16 offset, jit_code_t code)
d16005f8 501{
98fa08a5
PC		 502 struct regcache *reg_cache = state->reg_cache;
503 union code c = block->opcode_list[offset].c;
d16005f8 504 jit_state_t *_jit = block->_jit;
98fa08a5 505 u8 rd, rt, rs, temp, flags = 0;
d16005f8
PC		 506
507 jit_note(__FILE__, __LINE__);
d16005f8 508
98fa08a5
PC		 509 if (code == jit_code_rshr)
510 flags = REG_EXT;
511 else if (code == jit_code_rshr_u)
512 flags = REG_ZEXT;
d16005f8 513
684432ad
PC		 514 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
515 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
516 c.r.rt, c.r.rd, flags, flags, &rt, &rd);
d16005f8 517
684432ad 518 if (rt != rd) {
98fa08a5
PC		 519 jit_andi(rd, rs, 0x1f);
520 jit_new_node_www(code, rd, rt, rd);
521 } else {
522 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
523 jit_andi(temp, rs, 0x1f);
d16005f8 524 jit_new_node_www(code, rd, rt, temp);
98fa08a5
PC		 525 lightrec_free_reg(reg_cache, temp);
526 }
d16005f8
PC		 527
528 lightrec_free_reg(reg_cache, rs);
d16005f8
PC		 529 lightrec_free_reg(reg_cache, rt);
530 lightrec_free_reg(reg_cache, rd);
531}
532
02487de7
PC		 533static void rec_movi(struct lightrec_cstate *state,
534 const struct block *block, u16 offset)
535{
536 struct regcache *reg_cache = state->reg_cache;
537 union code c = block->opcode_list[offset].c;
538 jit_state_t *_jit = block->_jit;
539 u16 flags = REG_EXT;
684432ad 540 s32 value = (s32)(s16) c.i.imm;
02487de7
PC		 541 u8 rt;
542
684432ad
PC		 543 if (block->opcode_list[offset].flags & LIGHTREC_MOVI)
544 value += (s32)((u32)state->movi_temp[c.i.rt] << 16);
545
546 if (value >= 0)
02487de7
PC		 547 flags |= REG_ZEXT;
548
549 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
550
684432ad 551 jit_movi(rt, value);
02487de7
PC		 552
553 lightrec_free_reg(reg_cache, rt);
554}
555
98fa08a5
PC		 556static void rec_ADDIU(struct lightrec_cstate *state,
557 const struct block *block, u16 offset)
d16005f8 558{
684432ad
PC		 559 const struct opcode *op = &block->opcode_list[offset];
560
d16005f8 561 _jit_name(block->_jit, __func__);
02487de7 562
684432ad 563 if (op->i.rs && !(op->flags & LIGHTREC_MOVI))
02487de7
PC		 564 rec_alu_imm(state, block, offset, jit_code_addi, false);
565 else
566 rec_movi(state, block, offset);
d16005f8
PC		 567}
568
98fa08a5
PC		 569static void rec_ADDI(struct lightrec_cstate *state,
570 const struct block *block, u16 offset)
d16005f8
PC		 571{
572 /* TODO: Handle the exception? */
573 _jit_name(block->_jit, __func__);
02487de7 574 rec_ADDIU(state, block, offset);
d16005f8
PC		 575}
576
98fa08a5
PC		 577static void rec_SLTIU(struct lightrec_cstate *state,
578 const struct block *block, u16 offset)
d16005f8
PC		 579{
580 _jit_name(block->_jit, __func__);
98fa08a5 581 rec_alu_imm(state, block, offset, jit_code_lti_u, true);
d16005f8
PC		 582}
583
98fa08a5
PC		 584static void rec_SLTI(struct lightrec_cstate *state,
585 const struct block *block, u16 offset)
d16005f8
PC		 586{
587 _jit_name(block->_jit, __func__);
98fa08a5 588 rec_alu_imm(state, block, offset, jit_code_lti, true);
d16005f8
PC		 589}
590
98fa08a5
PC		 591static void rec_ANDI(struct lightrec_cstate *state,
592 const struct block *block, u16 offset)
d16005f8 593{
98fa08a5
PC		 594 struct regcache *reg_cache = state->reg_cache;
595 union code c = block->opcode_list[offset].c;
d16005f8
PC		 596 jit_state_t *_jit = block->_jit;
597 u8 rs, rt;
598
599 _jit_name(block->_jit, __func__);
600 jit_note(__FILE__, __LINE__);
684432ad
PC		 601
602 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
603 c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt);
d16005f8
PC		 604
605 /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
606 * casts to uint8_t / uint16_t. */
98fa08a5 607 if (c.i.imm == 0xff)
d16005f8 608 jit_extr_uc(rt, rs);
98fa08a5 609 else if (c.i.imm == 0xffff)
d16005f8
PC		 610 jit_extr_us(rt, rs);
611 else
98fa08a5
PC		 612 jit_andi(rt, rs, (u32)(u16) c.i.imm);
613
614 lightrec_free_reg(reg_cache, rs);
615 lightrec_free_reg(reg_cache, rt);
616}
617
618static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
619 u16 offset, jit_code_t code)
620{
621 struct regcache *reg_cache = state->reg_cache;
622 union code c = block->opcode_list[offset].c;
623 jit_state_t *_jit = block->_jit;
624 u8 rs, rt, flags;
625
626 jit_note(__FILE__, __LINE__);
684432ad
PC		 627
628 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
629 c.i.rs, c.i.rt, 0, 0, &rs, &rt);
98fa08a5
PC		 630
631 flags = lightrec_get_reg_in_flags(reg_cache, rs);
632 lightrec_set_reg_out_flags(reg_cache, rt, flags);
633
634 jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
d16005f8
PC		 635
636 lightrec_free_reg(reg_cache, rs);
637 lightrec_free_reg(reg_cache, rt);
638}
639
98fa08a5
PC		 640
641static void rec_ORI(struct lightrec_cstate *state,
642 const struct block *block, u16 offset)
d16005f8 643{
684432ad
PC		 644 const struct opcode *op = &block->opcode_list[offset];
645 struct regcache *reg_cache = state->reg_cache;
646 jit_state_t *_jit = block->_jit;
647 s32 val;
648 u8 rt;
649
650 _jit_name(_jit, __func__);
651
652 if (op->flags & LIGHTREC_MOVI) {
653 rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT);
654
655 val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm;
656 jit_movi(rt, val);
657
658 lightrec_free_reg(reg_cache, rt);
659 } else {
660 rec_alu_or_xor(state, block, offset, jit_code_ori);
661 }
d16005f8
PC		 662}
663
98fa08a5
PC		 664static void rec_XORI(struct lightrec_cstate *state,
665 const struct block *block, u16 offset)
d16005f8
PC		 666{
667 _jit_name(block->_jit, __func__);
98fa08a5 668 rec_alu_or_xor(state, block, offset, jit_code_xori);
d16005f8
PC		 669}
670
98fa08a5
PC		 671static void rec_LUI(struct lightrec_cstate *state,
672 const struct block *block, u16 offset)
d16005f8 673{
98fa08a5
PC		 674 struct regcache *reg_cache = state->reg_cache;
675 union code c = block->opcode_list[offset].c;
d16005f8 676 jit_state_t *_jit = block->_jit;
98fa08a5 677 u8 rt, flags = REG_EXT;
d16005f8 678
684432ad
PC		 679 if (block->opcode_list[offset].flags & LIGHTREC_MOVI) {
680 state->movi_temp[c.i.rt] = c.i.imm;
681 return;
682 }
683
d16005f8
PC		 684 jit_name(__func__);
685 jit_note(__FILE__, __LINE__);
d16005f8 686
98fa08a5
PC		 687 if (!(c.i.imm & BIT(15)))
688 flags |= REG_ZEXT;
689
690 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
691
692 jit_movi(rt, (s32)(c.i.imm << 16));
d16005f8
PC		 693
694 lightrec_free_reg(reg_cache, rt);
695}
696
98fa08a5
PC		 697static void rec_special_ADDU(struct lightrec_cstate *state,
698 const struct block *block, u16 offset)
d16005f8
PC		 699{
700 _jit_name(block->_jit, __func__);
98fa08a5 701 rec_alu_special(state, block, offset, jit_code_addr, false);
d16005f8
PC		 702}
703
98fa08a5
PC		 704static void rec_special_ADD(struct lightrec_cstate *state,
705 const struct block *block, u16 offset)
d16005f8
PC		 706{
707 /* TODO: Handle the exception? */
708 _jit_name(block->_jit, __func__);
98fa08a5 709 rec_alu_special(state, block, offset, jit_code_addr, false);
d16005f8
PC		 710}
711
98fa08a5
PC		 712static void rec_special_SUBU(struct lightrec_cstate *state,
713 const struct block *block, u16 offset)
d16005f8
PC		 714{
715 _jit_name(block->_jit, __func__);
98fa08a5 716 rec_alu_special(state, block, offset, jit_code_subr, false);
d16005f8
PC		 717}
718
98fa08a5
PC		 719static void rec_special_SUB(struct lightrec_cstate *state,
720 const struct block *block, u16 offset)
d16005f8
PC		 721{
722 /* TODO: Handle the exception? */
723 _jit_name(block->_jit, __func__);
98fa08a5 724 rec_alu_special(state, block, offset, jit_code_subr, false);
d16005f8
PC		 725}
726
98fa08a5
PC		 727static void rec_special_AND(struct lightrec_cstate *state,
728 const struct block *block, u16 offset)
d16005f8 729{
98fa08a5
PC		 730 struct regcache *reg_cache = state->reg_cache;
731 union code c = block->opcode_list[offset].c;
732 jit_state_t *_jit = block->_jit;
733 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
734
d16005f8 735 _jit_name(block->_jit, __func__);
98fa08a5 736 jit_note(__FILE__, __LINE__);
684432ad 737
98fa08a5 738 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 739 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
740 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
98fa08a5
PC		 741
742 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
743 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
744
745 /* Z(rd) = Z(rs) | Z(rt) */
746 flags_rd = REG_ZEXT & (flags_rs | flags_rt);
747
748 /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
749 if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
750 ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
751 (REG_EXT & flags_rs & flags_rt))
752 flags_rd |= REG_EXT;
753
754 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
755
756 jit_andr(rd, rs, rt);
757
758 lightrec_free_reg(reg_cache, rs);
759 lightrec_free_reg(reg_cache, rt);
760 lightrec_free_reg(reg_cache, rd);
d16005f8
PC		 761}
762
98fa08a5
PC		 763static void rec_special_or_nor(struct lightrec_cstate *state,
764 const struct block *block, u16 offset, bool nor)
765{
766 struct regcache *reg_cache = state->reg_cache;
767 union code c = block->opcode_list[offset].c;
768 jit_state_t *_jit = block->_jit;
769 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
770
771 jit_note(__FILE__, __LINE__);
684432ad 772
98fa08a5 773 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 774 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
775 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
98fa08a5
PC		 776
777 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
778 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
779
780 /* or: Z(rd) = Z(rs) & Z(rt)
781 * nor: Z(rd) = 0 */
782 if (!nor)
783 flags_rd = REG_ZEXT & flags_rs & flags_rt;
784
6ce0b00a
PC		 785 /* E(rd) = E(rs) & E(rt) */
786 if (REG_EXT & flags_rs & flags_rt)
98fa08a5
PC		 787 flags_rd |= REG_EXT;
788
789 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
790
791 jit_orr(rd, rs, rt);
792
793 if (nor)
794 jit_comr(rd, rd);
795
796 lightrec_free_reg(reg_cache, rs);
797 lightrec_free_reg(reg_cache, rt);
798 lightrec_free_reg(reg_cache, rd);
799}
800
801static void rec_special_OR(struct lightrec_cstate *state,
802 const struct block *block, u16 offset)
d16005f8
PC		 803{
804 _jit_name(block->_jit, __func__);
98fa08a5 805 rec_special_or_nor(state, block, offset, false);
d16005f8
PC		 806}
807
98fa08a5
PC		 808static void rec_special_NOR(struct lightrec_cstate *state,
809 const struct block *block, u16 offset)
d16005f8
PC		 810{
811 _jit_name(block->_jit, __func__);
98fa08a5 812 rec_special_or_nor(state, block, offset, true);
d16005f8
PC		 813}
814
98fa08a5
PC		 815static void rec_special_XOR(struct lightrec_cstate *state,
816 const struct block *block, u16 offset)
d16005f8 817{
98fa08a5
PC		 818 struct regcache *reg_cache = state->reg_cache;
819 union code c = block->opcode_list[offset].c;
d16005f8 820 jit_state_t *_jit = block->_jit;
98fa08a5 821 u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
d16005f8 822
98fa08a5
PC		 823 _jit_name(block->_jit, __func__);
824
825 jit_note(__FILE__, __LINE__);
684432ad 826
98fa08a5 827 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
684432ad
PC		 828 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
829 c.r.rs, c.r.rd, 0, 0, &rs, &rd);
d16005f8 830
98fa08a5
PC		 831 flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
832 flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
d16005f8 833
98fa08a5
PC		 834 /* Z(rd) = Z(rs) & Z(rt) */
835 flags_rd = REG_ZEXT & flags_rs & flags_rt;
836
837 /* E(rd) = E(rs) & E(rt) */
838 flags_rd |= REG_EXT & flags_rs & flags_rt;
839
840 lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
841
842 jit_xorr(rd, rs, rt);
843
844 lightrec_free_reg(reg_cache, rs);
845 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 846 lightrec_free_reg(reg_cache, rd);
847}
848
98fa08a5
PC		 849static void rec_special_SLTU(struct lightrec_cstate *state,
850 const struct block *block, u16 offset)
d16005f8
PC		 851{
852 _jit_name(block->_jit, __func__);
98fa08a5 853 rec_alu_special(state, block, offset, jit_code_ltr_u, true);
d16005f8
PC		 854}
855
98fa08a5
PC		 856static void rec_special_SLT(struct lightrec_cstate *state,
857 const struct block *block, u16 offset)
d16005f8
PC		 858{
859 _jit_name(block->_jit, __func__);
98fa08a5 860 rec_alu_special(state, block, offset, jit_code_ltr, true);
d16005f8
PC		 861}
862
98fa08a5
PC		 863static void rec_special_SLLV(struct lightrec_cstate *state,
864 const struct block *block, u16 offset)
d16005f8
PC		 865{
866 _jit_name(block->_jit, __func__);
98fa08a5 867 rec_alu_shiftv(state, block, offset, jit_code_lshr);
d16005f8
PC		 868}
869
98fa08a5
PC		 870static void rec_special_SRLV(struct lightrec_cstate *state,
871 const struct block *block, u16 offset)
d16005f8
PC		 872{
873 _jit_name(block->_jit, __func__);
98fa08a5 874 rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
d16005f8
PC		 875}
876
98fa08a5
PC		 877static void rec_special_SRAV(struct lightrec_cstate *state,
878 const struct block *block, u16 offset)
d16005f8
PC		 879{
880 _jit_name(block->_jit, __func__);
98fa08a5 881 rec_alu_shiftv(state, block, offset, jit_code_rshr);
d16005f8
PC		 882}
883
98fa08a5
PC		 884static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
885 u16 offset, jit_code_t code)
d16005f8 886{
98fa08a5
PC		 887 struct regcache *reg_cache = state->reg_cache;
888 union code c = block->opcode_list[offset].c;
d16005f8 889 jit_state_t *_jit = block->_jit;
684432ad 890 u8 rd, rt, flags = 0, out_flags = 0;
d16005f8
PC		 891
892 jit_note(__FILE__, __LINE__);
893
98fa08a5
PC		 894 if (code == jit_code_rshi)
895 flags = REG_EXT;
896 else if (code == jit_code_rshi_u)
897 flags = REG_ZEXT;
d16005f8 898
98fa08a5
PC		 899 /* Input reg is zero-extended, if we SRL at least by one bit, we know
900 * the output reg will be both zero-extended and sign-extended. */
684432ad 901 out_flags = flags;
98fa08a5 902 if (code == jit_code_rshi_u && c.r.imm)
684432ad
PC		 903 out_flags |= REG_EXT;
904
905 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
906 c.r.rt, c.r.rd, flags, out_flags, &rt, &rd);
98fa08a5
PC		 907
908 jit_new_node_www(code, rd, rt, c.r.imm);
d16005f8
PC		 909
910 lightrec_free_reg(reg_cache, rt);
911 lightrec_free_reg(reg_cache, rd);
912}
913
98fa08a5
PC		 914static void rec_special_SLL(struct lightrec_cstate *state,
915 const struct block *block, u16 offset)
d16005f8
PC		 916{
917 _jit_name(block->_jit, __func__);
98fa08a5 918 rec_alu_shift(state, block, offset, jit_code_lshi);
d16005f8
PC		 919}
920
98fa08a5
PC		 921static void rec_special_SRL(struct lightrec_cstate *state,
922 const struct block *block, u16 offset)
d16005f8
PC		 923{
924 _jit_name(block->_jit, __func__);
98fa08a5 925 rec_alu_shift(state, block, offset, jit_code_rshi_u);
d16005f8
PC		 926}
927
98fa08a5
PC		 928static void rec_special_SRA(struct lightrec_cstate *state,
929 const struct block *block, u16 offset)
d16005f8
PC		 930{
931 _jit_name(block->_jit, __func__);
98fa08a5 932 rec_alu_shift(state, block, offset, jit_code_rshi);
d16005f8
PC		 933}
934
98fa08a5
PC		 935static void rec_alu_mult(struct lightrec_cstate *state,
936 const struct block *block, u16 offset, bool is_signed)
d16005f8 937{
98fa08a5
PC		 938 struct regcache *reg_cache = state->reg_cache;
939 union code c = block->opcode_list[offset].c;
03535202 940 u32 flags = block->opcode_list[offset].flags;
98fa08a5
PC		 941 u8 reg_lo = get_mult_div_lo(c);
942 u8 reg_hi = get_mult_div_hi(c);
d16005f8 943 jit_state_t *_jit = block->_jit;
98fa08a5 944 u8 lo, hi, rs, rt, rflags = 0;
5459088b
PC		 945 bool no_lo = op_flag_no_lo(flags);
946 bool no_hi = op_flag_no_hi(flags);
d16005f8
PC		 947
948 jit_note(__FILE__, __LINE__);
949
98fa08a5
PC		 950 if (is_signed)
951 rflags = REG_EXT;
952 else
953 rflags = REG_ZEXT;
954
955 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
956 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
957
5459088b 958 if (!no_lo)
98fa08a5 959 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
98fa08a5 960
5459088b 961 if (!no_hi)
98fa08a5
PC		 962 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
963
964 if (__WORDSIZE == 32) {
965 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
966 * operation if the MULT was detected a 32-bit only. */
5459088b 967 if (no_lo) {
98fa08a5 968 if (is_signed)
5459088b 969 jit_hmulr(hi, rs, rt);
98fa08a5 970 else
5459088b
PC		 971 jit_hmulr_u(hi, rs, rt);
972 } else if (no_hi) {
98fa08a5 973 jit_mulr(lo, rs, rt);
5459088b
PC		 974 } else if (is_signed) {
975 jit_qmulr(lo, hi, rs, rt);
976 } else {
977 jit_qmulr_u(lo, hi, rs, rt);
98fa08a5 978 }
d16005f8 979 } else {
98fa08a5 980 /* On 64-bit systems, do a 64*64->64 bit operation. */
5459088b 981 if (no_lo) {
98fa08a5
PC		 982 jit_mulr(hi, rs, rt);
983 jit_rshi(hi, hi, 32);
984 } else {
985 jit_mulr(lo, rs, rt);
d16005f8 986
98fa08a5 987 /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
5459088b 988 if (!no_hi)
98fa08a5
PC		 989 jit_rshi(hi, lo, 32);
990 }
d16005f8
PC		 991 }
992
d16005f8
PC		 993 lightrec_free_reg(reg_cache, rs);
994 lightrec_free_reg(reg_cache, rt);
5459088b 995 if (!no_lo)
98fa08a5 996 lightrec_free_reg(reg_cache, lo);
5459088b 997 if (!no_hi)
d16005f8
PC		 998 lightrec_free_reg(reg_cache, hi);
999}
1000
98fa08a5
PC		 1001static void rec_alu_div(struct lightrec_cstate *state,
1002 const struct block *block, u16 offset, bool is_signed)
d16005f8 1003{
98fa08a5
PC		 1004 struct regcache *reg_cache = state->reg_cache;
1005 union code c = block->opcode_list[offset].c;
03535202
PC		 1006 u32 flags = block->opcode_list[offset].flags;
1007 bool no_check = op_flag_no_div_check(flags);
98fa08a5
PC		 1008 u8 reg_lo = get_mult_div_lo(c);
1009 u8 reg_hi = get_mult_div_hi(c);
d16005f8
PC		 1010 jit_state_t *_jit = block->_jit;
1011 jit_node_t *branch, *to_end;
fd58fa32 1012 u8 lo = 0, hi = 0, rs, rt, rflags = 0;
d16005f8
PC		 1013
1014 jit_note(__FILE__, __LINE__);
d16005f8 1015
98fa08a5
PC		 1016 if (is_signed)
1017 rflags = REG_EXT;
1018 else
1019 rflags = REG_ZEXT;
1020
1021 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
1022 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
1023
03535202 1024 if (!op_flag_no_lo(flags))
98fa08a5
PC		 1025 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
1026
03535202 1027 if (!op_flag_no_hi(flags))
98fa08a5 1028 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
d16005f8
PC		 1029
1030 /* Jump to special handler if dividing by zero */
98fa08a5
PC		 1031 if (!no_check)
1032 branch = jit_beqi(rt, 0);
d16005f8 1033
03535202 1034 if (op_flag_no_lo(flags)) {
98fa08a5
PC		 1035 if (is_signed)
1036 jit_remr(hi, rs, rt);
1037 else
1038 jit_remr_u(hi, rs, rt);
03535202 1039 } else if (op_flag_no_hi(flags)) {
98fa08a5
PC		 1040 if (is_signed)
1041 jit_divr(lo, rs, rt);
1042 else
1043 jit_divr_u(lo, rs, rt);
d16005f8 1044 } else {
98fa08a5
PC		 1045 if (is_signed)
1046 jit_qdivr(lo, hi, rs, rt);
1047 else
1048 jit_qdivr_u(lo, hi, rs, rt);
d16005f8 1049 }
d16005f8 1050
98fa08a5 1051 if (!no_check) {
98fa08a5 1052 /* Jump above the div-by-zero handler */
22eee2ac 1053 to_end = jit_b();
d16005f8 1054
98fa08a5
PC		 1055 jit_patch(branch);
1056
03535202 1057 if (!op_flag_no_lo(flags)) {
98fa08a5 1058 if (is_signed) {
9259d748 1059 jit_ltr(lo, rs, rt);
98fa08a5
PC		 1060 jit_lshi(lo, lo, 1);
1061 jit_subi(lo, lo, 1);
1062 } else {
9259d748 1063 jit_subi(lo, rt, 1);
98fa08a5
PC		 1064 }
1065 }
d16005f8 1066
03535202 1067 if (!op_flag_no_hi(flags))
98fa08a5 1068 jit_movr(hi, rs);
d16005f8 1069
98fa08a5
PC		 1070 jit_patch(to_end);
1071 }
d16005f8
PC		 1072
1073 lightrec_free_reg(reg_cache, rs);
1074 lightrec_free_reg(reg_cache, rt);
98fa08a5 1075
03535202 1076 if (!op_flag_no_lo(flags))
98fa08a5
PC		 1077 lightrec_free_reg(reg_cache, lo);
1078
03535202 1079 if (!op_flag_no_hi(flags))
98fa08a5 1080 lightrec_free_reg(reg_cache, hi);
d16005f8
PC		 1081}
1082
98fa08a5
PC		 1083static void rec_special_MULT(struct lightrec_cstate *state,
1084 const struct block *block, u16 offset)
d16005f8
PC		 1085{
1086 _jit_name(block->_jit, __func__);
98fa08a5 1087 rec_alu_mult(state, block, offset, true);
d16005f8
PC		 1088}
1089
98fa08a5
PC		 1090static void rec_special_MULTU(struct lightrec_cstate *state,
1091 const struct block *block, u16 offset)
d16005f8
PC		 1092{
1093 _jit_name(block->_jit, __func__);
98fa08a5 1094 rec_alu_mult(state, block, offset, false);
d16005f8
PC		 1095}
1096
98fa08a5
PC		 1097static void rec_special_DIV(struct lightrec_cstate *state,
1098 const struct block *block, u16 offset)
d16005f8
PC		 1099{
1100 _jit_name(block->_jit, __func__);
98fa08a5 1101 rec_alu_div(state, block, offset, true);
d16005f8
PC		 1102}
1103
98fa08a5
PC		 1104static void rec_special_DIVU(struct lightrec_cstate *state,
1105 const struct block *block, u16 offset)
d16005f8
PC		 1106{
1107 _jit_name(block->_jit, __func__);
98fa08a5 1108 rec_alu_div(state, block, offset, false);
d16005f8
PC		 1109}
1110
98fa08a5 1111static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
684432ad
PC		 1112 const struct block *block, u16 offset,
1113 u8 dst, u8 src)
d16005f8 1114{
98fa08a5 1115 struct regcache *reg_cache = state->reg_cache;
d16005f8
PC		 1116 jit_state_t *_jit = block->_jit;
1117
1118 jit_note(__FILE__, __LINE__);
684432ad
PC		 1119
1120 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
1121 src, dst, 0, REG_EXT, &src, &dst);
d16005f8 1122
d16005f8 1123 jit_extr_i(dst, src);
d16005f8
PC		 1124
1125 lightrec_free_reg(reg_cache, src);
1126 lightrec_free_reg(reg_cache, dst);
1127}
1128
98fa08a5
PC		 1129static void rec_special_MFHI(struct lightrec_cstate *state,
1130 const struct block *block, u16 offset)
d16005f8 1131{
98fa08a5
PC		 1132 union code c = block->opcode_list[offset].c;
1133
d16005f8 1134 _jit_name(block->_jit, __func__);
684432ad 1135 rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI);
d16005f8
PC		 1136}
1137
98fa08a5
PC		 1138static void rec_special_MTHI(struct lightrec_cstate *state,
1139 const struct block *block, u16 offset)
d16005f8 1140{
98fa08a5
PC		 1141 union code c = block->opcode_list[offset].c;
1142
d16005f8 1143 _jit_name(block->_jit, __func__);
684432ad 1144 rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs);
d16005f8
PC		 1145}
1146
98fa08a5
PC		 1147static void rec_special_MFLO(struct lightrec_cstate *state,
1148 const struct block *block, u16 offset)
d16005f8 1149{
98fa08a5
PC		 1150 union code c = block->opcode_list[offset].c;
1151
d16005f8 1152 _jit_name(block->_jit, __func__);
684432ad 1153 rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO);
d16005f8
PC		 1154}
1155
98fa08a5
PC		 1156static void rec_special_MTLO(struct lightrec_cstate *state,
1157 const struct block *block, u16 offset)
d16005f8 1158{
98fa08a5
PC		 1159 union code c = block->opcode_list[offset].c;
1160
d16005f8 1161 _jit_name(block->_jit, __func__);
684432ad 1162 rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs);
d16005f8
PC		 1163}
1164
ba3814c1
PC		 1165static void call_to_c_wrapper(struct lightrec_cstate *state,
1166 const struct block *block, u32 arg,
1167 enum c_wrappers wrapper)
d16005f8 1168{
98fa08a5 1169 struct regcache *reg_cache = state->reg_cache;
d16005f8 1170 jit_state_t *_jit = block->_jit;
ba3814c1 1171 s8 tmp, tmp2;
d16005f8 1172
ba3814c1
PC		 1173 /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1174 tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
d16005f8 1175
8afce295
PC		 1176 jit_movi(tmp2, (unsigned int)wrapper << (1 + __WORDSIZE / 32));
1177
ba3814c1 1178 tmp = lightrec_get_reg_with_value(reg_cache,
8afce295 1179 (intptr_t) state->state->c_wrapper);
ba3814c1
PC		 1180 if (tmp < 0) {
1181 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
8afce295 1182 jit_ldxi(tmp, LIGHTREC_REG_STATE, lightrec_offset(c_wrapper));
ba3814c1
PC		 1183
1184 lightrec_temp_set_value(reg_cache, tmp,
8afce295 1185 (intptr_t) state->state->c_wrapper);
22eee2ac
PC		 1186 }
1187
ba3814c1
PC		 1188 lightrec_free_reg(reg_cache, tmp2);
1189
1190#ifdef __mips__
1191 /* On MIPS, register t9 is always used as the target register for JALR.
1192 * Therefore if it does not contain the target address we must
1193 * invalidate it. */
1194 if (tmp != _T9)
1195 lightrec_unload_reg(reg_cache, _jit, _T9);
1196#endif
1197
1198 jit_prepare();
1199 jit_pushargi(arg);
1200
22eee2ac 1201 lightrec_regcache_mark_live(reg_cache, _jit);
98fa08a5 1202 jit_callr(tmp);
d16005f8 1203
98fa08a5 1204 lightrec_free_reg(reg_cache, tmp);
98fa08a5
PC		 1205 lightrec_regcache_mark_live(reg_cache, _jit);
1206}
1207
1208static void rec_io(struct lightrec_cstate *state,
1209 const struct block *block, u16 offset,
1210 bool load_rt, bool read_rt)
1211{
1212 struct regcache *reg_cache = state->reg_cache;
1213 jit_state_t *_jit = block->_jit;
1214 union code c = block->opcode_list[offset].c;
03535202 1215 u32 flags = block->opcode_list[offset].flags;
22eee2ac 1216 bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
98fa08a5 1217 u32 lut_entry;
cb72ea13 1218 u8 zero;
d16005f8 1219
98fa08a5 1220 jit_note(__FILE__, __LINE__);
d16005f8 1221
98fa08a5
PC		 1222 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1223
1224 if (read_rt && likely(c.i.rt))
1225 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
d16005f8 1226 else if (load_rt)
98fa08a5 1227 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
d16005f8 1228
cb72ea13
PC		 1229 if (op_flag_load_delay(flags) && !state->no_load_delay) {
1230 /* Clear state->in_delay_slot_n. This notifies the lightrec_rw
1231 * wrapper that it should write the REG_TEMP register instead of
1232 * the actual output register of the opcode. */
1233 zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
8afce295
PC		 1234 jit_stxi_c(lightrec_offset(in_delay_slot_n),
1235 LIGHTREC_REG_STATE, zero);
cb72ea13
PC		 1236 lightrec_free_reg(reg_cache, zero);
1237 }
1238
d16005f8 1239 if (is_tagged) {
ba3814c1 1240 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
d16005f8 1241 } else {
98fa08a5
PC		 1242 lut_entry = lightrec_get_lut_entry(block);
1243 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
ba3814c1 1244 C_WRAPPER_RW_GENERIC);
d16005f8 1245 }
d16005f8
PC		 1246}
1247
f5ee77ca 1248static u32 rec_ram_mask(const struct lightrec_state *state)
02487de7
PC		 1249{
1250 return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1251}
1252
13b02197
PC		 1253static u32 rec_io_mask(const struct lightrec_state *state)
1254{
1255 u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1256
2ec79b77 1257 return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0);
13b02197
PC		 1258}
1259
878e6cda
PC		 1260static void rec_add_offset(struct lightrec_cstate *cstate,
1261 jit_state_t *_jit, u8 reg_out, u8 reg_in,
1262 uintptr_t offset)
1263{
1264 struct regcache *reg_cache = cstate->reg_cache;
1265 u8 reg_imm;
1266
1267 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, offset);
1268 jit_addr(reg_out, reg_in, reg_imm);
1269
1270 lightrec_free_reg(reg_cache, reg_imm);
1271}
1272
1273static void rec_and_mask(struct lightrec_cstate *cstate,
1274 jit_state_t *_jit, u8 reg_out, u8 reg_in, u32 mask)
1275{
1276 struct regcache *reg_cache = cstate->reg_cache;
1277 u8 reg_imm;
1278
1279 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1280 jit_andr(reg_out, reg_in, reg_imm);
1281
1282 lightrec_free_reg(reg_cache, reg_imm);
1283}
1284
22eee2ac
PC		 1285static void rec_store_memory(struct lightrec_cstate *cstate,
1286 const struct block *block,
1287 u16 offset, jit_code_t code,
878e6cda
PC		 1288 jit_code_t swap_code, uintptr_t addr_offset,
1289 u32 addr_mask, bool invalidate)
22eee2ac 1290{
02487de7 1291 const struct lightrec_state *state = cstate->state;
22eee2ac
PC		 1292 struct regcache *reg_cache = cstate->reg_cache;
1293 struct opcode *op = &block->opcode_list[offset];
1294 jit_state_t *_jit = block->_jit;
1295 union code c = op->c;
f5ee77ca 1296 u8 rs, rt, tmp = 0, tmp2 = 0, tmp3, addr_reg, addr_reg2;
22eee2ac 1297 s16 imm = (s16)c.i.imm;
02487de7 1298 s32 simm = (s32)imm << (1 - lut_is_32bit(state));
8afce295 1299 s32 lut_offt = lightrec_offset(code_lut);
03535202 1300 bool no_mask = op_flag_no_mask(op->flags);
02487de7 1301 bool add_imm = c.i.imm &&
5459088b
PC		 1302 (c.i.op == OP_META_SWU
1303 || (!state->mirrors_mapped && !no_mask) || (invalidate &&
02487de7 1304 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
684432ad 1305 bool need_tmp = !no_mask || add_imm || invalidate;
9259d748 1306 bool swc2 = c.i.op == OP_SWC2;
cb72ea13 1307 u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
22eee2ac 1308
22eee2ac
PC		 1309 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1310 if (need_tmp)
1311 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1312
1313 addr_reg = rs;
1314
1315 if (add_imm) {
1316 jit_addi(tmp, addr_reg, (s16)c.i.imm);
0e720fb1 1317 lightrec_free_reg(reg_cache, rs);
22eee2ac
PC		 1318 addr_reg = tmp;
1319 imm = 0;
1320 } else if (simm) {
1321 lut_offt += simm;
1322 }
1323
1324 if (!no_mask) {
878e6cda 1325 rec_and_mask(cstate, _jit, tmp, addr_reg, addr_mask);
22eee2ac
PC		 1326 addr_reg = tmp;
1327 }
1328
22eee2ac 1329 if (addr_offset) {
ba3814c1 1330 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
878e6cda 1331 rec_add_offset(cstate, _jit, tmp2, addr_reg, addr_offset);
22eee2ac
PC		 1332 addr_reg2 = tmp2;
1333 } else {
1334 addr_reg2 = addr_reg;
1335 }
1336
684432ad
PC		 1337 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1338
9259d748 1339 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1340 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1341
1342 jit_new_node_ww(swap_code, tmp3, rt);
5459088b
PC		 1343
1344 if (c.i.op == OP_META_SWU)
1345 jit_unstr(addr_reg2, tmp3, LIGHTNING_UNALIGNED_32BIT);
1346 else
1347 jit_new_node_www(code, imm, addr_reg2, tmp3);
02487de7
PC		 1348
1349 lightrec_free_reg(reg_cache, tmp3);
5459088b
PC		 1350 } else if (c.i.op == OP_META_SWU) {
1351 jit_unstr(addr_reg2, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1352 } else {
1353 jit_new_node_www(code, imm, addr_reg2, rt);
1354 }
1355
22eee2ac
PC		 1356 lightrec_free_reg(reg_cache, rt);
1357
1358 if (invalidate) {
1359 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1360
1361 if (c.i.op != OP_SW) {
ba3814c1
PC		 1362 jit_andi(tmp, addr_reg, ~3);
1363 addr_reg = tmp;
22eee2ac
PC		 1364 }
1365
02487de7 1366 if (!lut_is_32bit(state)) {
ba3814c1
PC		 1367 jit_lshi(tmp, addr_reg, 1);
1368 addr_reg = tmp;
22eee2ac
PC		 1369 }
1370
02487de7
PC		 1371 if (addr_reg == rs && c.i.rs == 0) {
1372 addr_reg = LIGHTREC_REG_STATE;
1373 } else {
cb72ea13 1374 jit_add_state(tmp, addr_reg);
ba3814c1 1375 addr_reg = tmp;
22eee2ac
PC		 1376 }
1377
02487de7
PC		 1378 if (lut_is_32bit(state))
1379 jit_stxi_i(lut_offt, addr_reg, tmp3);
1380 else
1381 jit_stxi(lut_offt, addr_reg, tmp3);
22eee2ac
PC		 1382
1383 lightrec_free_reg(reg_cache, tmp3);
1384 }
1385
ba3814c1 1386 if (addr_offset)
22eee2ac
PC		 1387 lightrec_free_reg(reg_cache, tmp2);
1388 if (need_tmp)
1389 lightrec_free_reg(reg_cache, tmp);
1390 lightrec_free_reg(reg_cache, rs);
1391}
1392
1393static void rec_store_ram(struct lightrec_cstate *cstate,
1394 const struct block *block,
1395 u16 offset, jit_code_t code,
02487de7 1396 jit_code_t swap_code, bool invalidate)
22eee2ac 1397{
f5ee77ca 1398 const struct lightrec_state *state = cstate->state;
02487de7 1399
22eee2ac
PC		 1400 _jit_note(block->_jit, __FILE__, __LINE__);
1401
02487de7
PC		 1402 return rec_store_memory(cstate, block, offset, code, swap_code,
1403 state->offset_ram, rec_ram_mask(state),
1404 invalidate);
22eee2ac
PC		 1405}
1406
1407static void rec_store_scratch(struct lightrec_cstate *cstate,
02487de7
PC		 1408 const struct block *block, u16 offset,
1409 jit_code_t code, jit_code_t swap_code)
22eee2ac
PC		 1410{
1411 _jit_note(block->_jit, __FILE__, __LINE__);
1412
02487de7 1413 return rec_store_memory(cstate, block, offset, code, swap_code,
22eee2ac
PC		 1414 cstate->state->offset_scratch,
1415 0x1fffffff, false);
1416}
1417
ba3814c1
PC		 1418static void rec_store_io(struct lightrec_cstate *cstate,
1419 const struct block *block, u16 offset,
1420 jit_code_t code, jit_code_t swap_code)
1421{
1422 _jit_note(block->_jit, __FILE__, __LINE__);
1423
1424 return rec_store_memory(cstate, block, offset, code, swap_code,
1425 cstate->state->offset_io,
13b02197 1426 rec_io_mask(cstate->state), false);
ba3814c1
PC		 1427}
1428
98fa08a5
PC		 1429static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1430 const struct block *block,
02487de7
PC		 1431 u16 offset, jit_code_t code,
1432 jit_code_t swap_code)
d16005f8 1433{
f5ee77ca 1434 const struct lightrec_state *state = cstate->state;
878e6cda 1435 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
98fa08a5
PC		 1436 struct regcache *reg_cache = cstate->reg_cache;
1437 union code c = block->opcode_list[offset].c;
d16005f8
PC		 1438 jit_state_t *_jit = block->_jit;
1439 jit_node_t *to_not_ram, *to_end;
9259d748 1440 bool swc2 = c.i.op == OP_SWC2;
878e6cda 1441 u8 addr_reg, tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt;
d16005f8
PC		 1442 s16 imm;
1443
1444 jit_note(__FILE__, __LINE__);
98fa08a5 1445 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
d16005f8 1446 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5 1447
d16005f8 1448 /* Convert to KUNSEG and avoid RAM mirrors */
5459088b 1449 if ((c.i.op == OP_META_SWU || !state->mirrors_mapped) && c.i.imm) {
d16005f8 1450 imm = 0;
98fa08a5 1451 jit_addi(tmp, rs, (s16)c.i.imm);
878e6cda 1452 addr_reg = tmp;
d16005f8 1453 } else {
684432ad 1454 imm = (s16)c.i.imm;
878e6cda 1455 addr_reg = rs;
d16005f8
PC		 1456 }
1457
878e6cda
PC		 1458 rec_and_mask(cstate, _jit, tmp, addr_reg, 0x1f800000 | (ram_size - 1));
1459
d16005f8
PC		 1460 lightrec_free_reg(reg_cache, rs);
1461
1462 if (state->offset_ram != state->offset_scratch) {
684432ad
PC		 1463 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1464
d16005f8
PC		 1465 to_not_ram = jit_bmsi(tmp, BIT(28));
1466
1467 jit_movi(tmp2, state->offset_ram);
1468
22eee2ac 1469 to_end = jit_b();
d16005f8
PC		 1470 jit_patch(to_not_ram);
1471
1472 jit_movi(tmp2, state->offset_scratch);
1473 jit_patch(to_end);
1474 } else if (state->offset_ram) {
684432ad
PC		 1475 tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1476 state->offset_ram);
d16005f8
PC		 1477 }
1478
684432ad 1479 if (state->offset_ram || state->offset_scratch) {
d16005f8 1480 jit_addr(tmp, tmp, tmp2);
98fa08a5
PC		 1481 lightrec_free_reg(reg_cache, tmp2);
1482 }
d16005f8 1483
9259d748 1484 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
ba3814c1 1485
9259d748 1486 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1487 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1488
1489 jit_new_node_ww(swap_code, tmp2, rt);
5459088b
PC		 1490
1491 if (c.i.op == OP_META_SWU)
1492 jit_unstr(tmp, tmp2, LIGHTNING_UNALIGNED_32BIT);
1493 else
1494 jit_new_node_www(code, imm, tmp, tmp2);
02487de7
PC		 1495
1496 lightrec_free_reg(reg_cache, tmp2);
5459088b
PC		 1497 } else if (c.i.op == OP_META_SWU) {
1498 jit_unstr(tmp, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1499 } else {
1500 jit_new_node_www(code, imm, tmp, rt);
1501 }
d16005f8
PC		 1502
1503 lightrec_free_reg(reg_cache, rt);
1504 lightrec_free_reg(reg_cache, tmp);
1505}
1506
98fa08a5 1507static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
02487de7 1508 u16 offset, jit_code_t code, jit_code_t swap_code)
d16005f8 1509{
f5ee77ca 1510 const struct lightrec_state *state = cstate->state;
98fa08a5
PC		 1511 u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1512 struct regcache *reg_cache = cstate->reg_cache;
1513 union code c = block->opcode_list[offset].c;
d16005f8 1514 jit_state_t *_jit = block->_jit;
98fa08a5 1515 jit_node_t *to_not_ram, *to_end;
9259d748 1516 bool swc2 = c.i.op == OP_SWC2;
878e6cda 1517 u8 addr_reg, tmp, tmp2, tmp3, rs, rt, reg_imm;
cb72ea13 1518 u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
878e6cda 1519 u32 mask;
f5ee77ca 1520 bool different_offsets = state->offset_ram != state->offset_scratch;
d16005f8
PC		 1521
1522 jit_note(__FILE__, __LINE__);
1523
98fa08a5 1524 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
d16005f8 1525 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5 1526 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
d16005f8
PC		 1527
1528 /* Convert to KUNSEG and avoid RAM mirrors */
98fa08a5
PC		 1529 if (c.i.imm) {
1530 jit_addi(tmp2, rs, (s16)c.i.imm);
878e6cda 1531 addr_reg = tmp2;
d16005f8 1532 } else {
878e6cda 1533 addr_reg = rs;
d16005f8
PC		 1534 }
1535
878e6cda
PC		 1536 rec_and_mask(cstate, _jit, tmp2, addr_reg, 0x1f800000 | (ram_size - 1));
1537
d16005f8
PC		 1538 lightrec_free_reg(reg_cache, rs);
1539 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1540
878e6cda
PC		 1541 mask = c.i.op == OP_SW ? RAM_SIZE - 1 : (RAM_SIZE - 1) & ~3;
1542 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1543
f5ee77ca 1544 if (different_offsets) {
9259d748 1545 to_not_ram = jit_bgti(tmp2, ram_size);
878e6cda 1546 addr_reg = tmp2;
9259d748
PC		 1547 } else {
1548 jit_lti_u(tmp, tmp2, ram_size);
1549 jit_movnr(tmp, tmp2, tmp);
878e6cda 1550 addr_reg = tmp;
9259d748 1551 }
98fa08a5 1552
d16005f8 1553 /* Compute the offset to the code LUT */
878e6cda 1554 jit_andr(tmp, addr_reg, reg_imm);
9259d748 1555
02487de7 1556 if (!lut_is_32bit(state))
98fa08a5 1557 jit_lshi(tmp, tmp, 1);
cb72ea13 1558 jit_add_state(tmp, tmp);
d16005f8
PC		 1559
1560 /* Write NULL to the code LUT to invalidate any block that's there */
02487de7 1561 if (lut_is_32bit(state))
8afce295 1562 jit_stxi_i(lightrec_offset(code_lut), tmp, tmp3);
02487de7 1563 else
8afce295 1564 jit_stxi(lightrec_offset(code_lut), tmp, tmp3);
d16005f8 1565
5459088b
PC		 1566 if (c.i.op == OP_META_SWU) {
1567 /* With a SWU opcode, we might have touched the following 32-bit
1568 * word, so invalidate it as well */
1569 if (lut_is_32bit(state)) {
8afce295 1570 jit_stxi_i(lightrec_offset(code_lut) + 4, tmp, tmp3);
5459088b 1571 } else {
8afce295
PC		 1572 jit_stxi(lightrec_offset(code_lut) + sizeof(uintptr_t),
1573 tmp, tmp3);
5459088b
PC		 1574 }
1575 }
1576
f5ee77ca 1577 if (different_offsets) {
d16005f8
PC		 1578 jit_movi(tmp, state->offset_ram);
1579
22eee2ac 1580 to_end = jit_b();
9259d748 1581 jit_patch(to_not_ram);
d16005f8
PC		 1582 }
1583
d16005f8
PC		 1584 if (state->offset_ram || state->offset_scratch)
1585 jit_movi(tmp, state->offset_scratch);
1586
f5ee77ca 1587 if (different_offsets)
d16005f8
PC		 1588 jit_patch(to_end);
1589
1590 if (state->offset_ram || state->offset_scratch)
1591 jit_addr(tmp2, tmp2, tmp);
1592
1593 lightrec_free_reg(reg_cache, tmp);
1594 lightrec_free_reg(reg_cache, tmp3);
878e6cda 1595 lightrec_free_reg(reg_cache, reg_imm);
d16005f8 1596
9259d748 1597 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
02487de7 1598
9259d748 1599 if (is_big_endian() && swap_code && in_reg) {
02487de7
PC		 1600 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1601
1602 jit_new_node_ww(swap_code, tmp, rt);
5459088b
PC		 1603
1604 if (c.i.op == OP_META_SWU)
1605 jit_unstr(tmp2, tmp, LIGHTNING_UNALIGNED_32BIT);
1606 else
1607 jit_new_node_www(code, 0, tmp2, tmp);
02487de7
PC		 1608
1609 lightrec_free_reg(reg_cache, tmp);
5459088b
PC		 1610 } else if (c.i.op == OP_META_SWU) {
1611 jit_unstr(tmp2, rt, LIGHTNING_UNALIGNED_32BIT);
02487de7
PC		 1612 } else {
1613 jit_new_node_www(code, 0, tmp2, rt);
1614 }
d16005f8
PC		 1615
1616 lightrec_free_reg(reg_cache, rt);
1617 lightrec_free_reg(reg_cache, tmp2);
1618}
1619
98fa08a5 1620static void rec_store(struct lightrec_cstate *state,
02487de7
PC		 1621 const struct block *block, u16 offset,
1622 jit_code_t code, jit_code_t swap_code)
d16005f8 1623{
03535202 1624 u32 flags = block->opcode_list[offset].flags;
9259d748 1625 u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
03535202 1626 bool no_invalidate = op_flag_no_invalidate(flags) ||
684432ad 1627 (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY);
9259d748
PC		 1628 union code c = block->opcode_list[offset].c;
1629 bool is_swc2 = c.i.op == OP_SWC2;
1630
1631 if (is_swc2) {
1632 switch (mode) {
1633 case LIGHTREC_IO_RAM:
1634 case LIGHTREC_IO_SCRATCH:
1635 case LIGHTREC_IO_DIRECT:
1636 case LIGHTREC_IO_DIRECT_HW:
cb72ea13 1637 rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP);
9259d748
PC		 1638 break;
1639 default:
1640 break;
1641 }
1642 }
22eee2ac 1643
9259d748 1644 switch (mode) {
22eee2ac 1645 case LIGHTREC_IO_RAM:
02487de7
PC		 1646 rec_store_ram(state, block, offset, code,
1647 swap_code, !no_invalidate);
22eee2ac
PC		 1648 break;
1649 case LIGHTREC_IO_SCRATCH:
02487de7 1650 rec_store_scratch(state, block, offset, code, swap_code);
22eee2ac
PC		 1651 break;
1652 case LIGHTREC_IO_DIRECT:
02487de7
PC		 1653 if (no_invalidate) {
1654 rec_store_direct_no_invalidate(state, block, offset,
1655 code, swap_code);
1656 } else {
1657 rec_store_direct(state, block, offset, code, swap_code);
1658 }
22eee2ac 1659 break;
ba3814c1
PC		 1660 case LIGHTREC_IO_DIRECT_HW:
1661 rec_store_io(state, block, offset, code, swap_code);
1662 break;
22eee2ac 1663 default:
98fa08a5 1664 rec_io(state, block, offset, true, false);
9259d748 1665 return;
d16005f8 1666 }
9259d748
PC		 1667
1668 if (is_swc2)
cb72ea13 1669 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
d16005f8
PC		 1670}
1671
98fa08a5
PC		 1672static void rec_SB(struct lightrec_cstate *state,
1673 const struct block *block, u16 offset)
d16005f8
PC		 1674{
1675 _jit_name(block->_jit, __func__);
02487de7 1676 rec_store(state, block, offset, jit_code_stxi_c, 0);
d16005f8
PC		 1677}
1678
98fa08a5
PC		 1679static void rec_SH(struct lightrec_cstate *state,
1680 const struct block *block, u16 offset)
d16005f8
PC		 1681{
1682 _jit_name(block->_jit, __func__);
02487de7
PC		 1683 rec_store(state, block, offset,
1684 jit_code_stxi_s, jit_code_bswapr_us);
d16005f8
PC		 1685}
1686
98fa08a5
PC		 1687static void rec_SW(struct lightrec_cstate *state,
1688 const struct block *block, u16 offset)
1689
d16005f8 1690{
9259d748
PC		 1691 union code c = block->opcode_list[offset].c;
1692
1693 _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
02487de7
PC		 1694 rec_store(state, block, offset,
1695 jit_code_stxi_i, jit_code_bswapr_ui);
d16005f8
PC		 1696}
1697
98fa08a5
PC		 1698static void rec_SWL(struct lightrec_cstate *state,
1699 const struct block *block, u16 offset)
d16005f8
PC		 1700{
1701 _jit_name(block->_jit, __func__);
98fa08a5 1702 rec_io(state, block, offset, true, false);
d16005f8
PC		 1703}
1704
98fa08a5
PC		 1705static void rec_SWR(struct lightrec_cstate *state,
1706 const struct block *block, u16 offset)
d16005f8
PC		 1707{
1708 _jit_name(block->_jit, __func__);
98fa08a5 1709 rec_io(state, block, offset, true, false);
d16005f8
PC		 1710}
1711
22eee2ac 1712static void rec_load_memory(struct lightrec_cstate *cstate,
02487de7
PC		 1713 const struct block *block, u16 offset,
1714 jit_code_t code, jit_code_t swap_code, bool is_unsigned,
22eee2ac
PC		 1715 uintptr_t addr_offset, u32 addr_mask)
1716{
878e6cda 1717 struct lightrec_state *state = cstate->state;
22eee2ac
PC		 1718 struct regcache *reg_cache = cstate->reg_cache;
1719 struct opcode *op = &block->opcode_list[offset];
cb72ea13 1720 bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
22eee2ac 1721 jit_state_t *_jit = block->_jit;
9259d748 1722 u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
03535202 1723 bool no_mask = op_flag_no_mask(op->flags);
22eee2ac 1724 union code c = op->c;
02487de7 1725 s16 imm;
22eee2ac 1726
cb72ea13
PC		 1727 if (load_delay || c.i.op == OP_LWC2)
1728 out_reg = REG_TEMP;
9259d748
PC		 1729 else if (c.i.rt)
1730 out_reg = c.i.rt;
1731 else
22eee2ac
PC		 1732 return;
1733
1734 if (is_unsigned)
1735 flags |= REG_ZEXT;
1736
1737 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
9259d748 1738 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
22eee2ac 1739
5459088b 1740 if ((op->i.op == OP_META_LWU && c.i.imm)
878e6cda 1741 || (!state->mirrors_mapped && c.i.imm && !no_mask)) {
02487de7 1742 jit_addi(rt, rs, (s16)c.i.imm);
22eee2ac 1743 addr_reg = rt;
02487de7 1744 imm = 0;
22eee2ac
PC		 1745 } else {
1746 addr_reg = rs;
02487de7
PC		 1747 imm = (s16)c.i.imm;
1748 }
1749
5459088b
PC		 1750 if (op->i.op == OP_META_LWU)
1751 imm = LIGHTNING_UNALIGNED_32BIT;
1752
02487de7 1753 if (!no_mask) {
878e6cda 1754 rec_and_mask(cstate, _jit, rt, addr_reg, addr_mask);
02487de7 1755 addr_reg = rt;
22eee2ac
PC		 1756 }
1757
1758 if (addr_offset) {
878e6cda 1759 rec_add_offset(cstate, _jit, rt, addr_reg, addr_offset);
22eee2ac
PC		 1760 addr_reg = rt;
1761 }
1762
02487de7
PC		 1763 jit_new_node_www(code, rt, addr_reg, imm);
1764
1765 if (is_big_endian() && swap_code) {
1766 jit_new_node_ww(swap_code, rt, rt);
1767
1768 if (c.i.op == OP_LH)
1769 jit_extr_s(rt, rt);
1770 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1771 jit_extr_i(rt, rt);
1772 }
22eee2ac
PC		 1773
1774 lightrec_free_reg(reg_cache, rs);
1775 lightrec_free_reg(reg_cache, rt);
1776}
1777
1778static void rec_load_ram(struct lightrec_cstate *cstate,
02487de7
PC		 1779 const struct block *block, u16 offset,
1780 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1781{
1782 _jit_note(block->_jit, __FILE__, __LINE__);
1783
02487de7
PC		 1784 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1785 cstate->state->offset_ram, rec_ram_mask(cstate->state));
22eee2ac
PC		 1786}
1787
1788static void rec_load_bios(struct lightrec_cstate *cstate,
02487de7
PC		 1789 const struct block *block, u16 offset,
1790 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1791{
1792 _jit_note(block->_jit, __FILE__, __LINE__);
1793
02487de7 1794 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
22eee2ac
PC		 1795 cstate->state->offset_bios, 0x1fffffff);
1796}
1797
1798static void rec_load_scratch(struct lightrec_cstate *cstate,
02487de7
PC		 1799 const struct block *block, u16 offset,
1800 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
22eee2ac
PC		 1801{
1802 _jit_note(block->_jit, __FILE__, __LINE__);
1803
02487de7 1804 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
22eee2ac
PC		 1805 cstate->state->offset_scratch, 0x1fffffff);
1806}
1807
ba3814c1
PC		 1808static void rec_load_io(struct lightrec_cstate *cstate,
1809 const struct block *block, u16 offset,
1810 jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1811{
1812 _jit_note(block->_jit, __FILE__, __LINE__);
1813
1814 rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
13b02197 1815 cstate->state->offset_io, rec_io_mask(cstate->state));
ba3814c1
PC		 1816}
1817
02487de7
PC		 1818static void rec_load_direct(struct lightrec_cstate *cstate,
1819 const struct block *block, u16 offset,
1820 jit_code_t code, jit_code_t swap_code,
1821 bool is_unsigned)
d16005f8 1822{
f5ee77ca 1823 const struct lightrec_state *state = cstate->state;
98fa08a5 1824 struct regcache *reg_cache = cstate->reg_cache;
cb72ea13
PC		 1825 struct opcode *op = &block->opcode_list[offset];
1826 bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
d16005f8 1827 jit_state_t *_jit = block->_jit;
98fa08a5 1828 jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
9259d748 1829 u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
f5ee77ca 1830 bool different_offsets = state->offset_bios != state->offset_scratch;
cb72ea13 1831 union code c = op->c;
684432ad
PC		 1832 s32 addr_mask;
1833 u32 reg_imm;
1834 s8 offt_reg;
d16005f8
PC		 1835 s16 imm;
1836
cb72ea13
PC		 1837 if (load_delay || c.i.op == OP_LWC2)
1838 out_reg = REG_TEMP;
9259d748
PC		 1839 else if (c.i.rt)
1840 out_reg = c.i.rt;
1841 else
d16005f8
PC		 1842 return;
1843
98fa08a5
PC		 1844 if (is_unsigned)
1845 flags |= REG_ZEXT;
1846
d16005f8 1847 jit_note(__FILE__, __LINE__);
98fa08a5 1848 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
9259d748 1849 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
d16005f8
PC		 1850
1851 if ((state->offset_ram == state->offset_bios &&
1852 state->offset_ram == state->offset_scratch &&
5459088b
PC		 1853 state->mirrors_mapped && c.i.op != OP_META_LWU)
1854 || !c.i.imm) {
d16005f8 1855 addr_reg = rs;
98fa08a5 1856 imm = (s16)c.i.imm;
d16005f8 1857 } else {
98fa08a5 1858 jit_addi(rt, rs, (s16)c.i.imm);
d16005f8
PC		 1859 addr_reg = rt;
1860 imm = 0;
1861
98fa08a5 1862 if (c.i.rs != c.i.rt)
d16005f8
PC		 1863 lightrec_free_reg(reg_cache, rs);
1864 }
1865
5459088b
PC		 1866 if (op->i.op == OP_META_LWU)
1867 imm = LIGHTNING_UNALIGNED_32BIT;
1868
d16005f8
PC		 1869 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1870
1871 if (state->offset_ram == state->offset_bios &&
1872 state->offset_ram == state->offset_scratch) {
684432ad
PC		 1873 if (!state->mirrors_mapped)
1874 addr_mask = 0x1f800000 | (RAM_SIZE - 1);
1875 else
1876 addr_mask = 0x1fffffff;
1877
1878 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1879 addr_mask);
d16005f8
PC		 1880 if (!state->mirrors_mapped) {
1881 jit_andi(tmp, addr_reg, BIT(28));
1882 jit_rshi_u(tmp, tmp, 28 - 22);
684432ad 1883 jit_orr(tmp, tmp, reg_imm);
d16005f8
PC		 1884 jit_andr(rt, addr_reg, tmp);
1885 } else {
684432ad 1886 jit_andr(rt, addr_reg, reg_imm);
d16005f8
PC		 1887 }
1888
684432ad
PC		 1889 lightrec_free_reg(reg_cache, reg_imm);
1890
1891 if (state->offset_ram) {
1892 offt_reg = lightrec_get_reg_with_value(reg_cache,
1893 state->offset_ram);
1894 if (offt_reg < 0) {
1895 jit_movi(tmp, state->offset_ram);
1896 lightrec_temp_set_value(reg_cache, tmp,
1897 state->offset_ram);
1898 } else {
1899 lightrec_free_reg(reg_cache, tmp);
1900 tmp = offt_reg;
1901 }
1902 }
d16005f8
PC		 1903 } else {
1904 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1905
1906 /* Convert to KUNSEG and avoid RAM mirrors */
1907 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1908
1909 if (state->offset_ram)
1910 jit_movi(tmp, state->offset_ram);
1911
22eee2ac 1912 to_end = jit_b();
d16005f8
PC		 1913
1914 jit_patch(to_not_ram);
1915
f5ee77ca 1916 if (different_offsets)
d16005f8
PC		 1917 to_not_bios = jit_bmci(addr_reg, BIT(22));
1918
1919 /* Convert to KUNSEG */
1920 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1921
1922 jit_movi(tmp, state->offset_bios);
1923
f5ee77ca 1924 if (different_offsets) {
22eee2ac 1925 to_end2 = jit_b();
d16005f8
PC		 1926
1927 jit_patch(to_not_bios);
1928
1929 /* Convert to KUNSEG */
1930 jit_andi(rt, addr_reg, 0x1f800fff);
1931
1932 if (state->offset_scratch)
1933 jit_movi(tmp, state->offset_scratch);
1934
1935 jit_patch(to_end2);
1936 }
1937
1938 jit_patch(to_end);
1939 }
1940
1941 if (state->offset_ram || state->offset_bios || state->offset_scratch)
1942 jit_addr(rt, rt, tmp);
1943
1944 jit_new_node_www(code, rt, rt, imm);
1945
02487de7
PC		 1946 if (is_big_endian() && swap_code) {
1947 jit_new_node_ww(swap_code, rt, rt);
1948
1949 if (c.i.op == OP_LH)
1950 jit_extr_s(rt, rt);
1951 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1952 jit_extr_i(rt, rt);
1953 }
1954
d16005f8
PC		 1955 lightrec_free_reg(reg_cache, addr_reg);
1956 lightrec_free_reg(reg_cache, rt);
1957 lightrec_free_reg(reg_cache, tmp);
1958}
1959
98fa08a5 1960static void rec_load(struct lightrec_cstate *state, const struct block *block,
02487de7
PC		 1961 u16 offset, jit_code_t code, jit_code_t swap_code,
1962 bool is_unsigned)
d16005f8 1963{
9259d748
PC		 1964 const struct opcode *op = &block->opcode_list[offset];
1965 u32 flags = op->flags;
98fa08a5 1966
22eee2ac
PC		 1967 switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1968 case LIGHTREC_IO_RAM:
02487de7 1969 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1970 break;
1971 case LIGHTREC_IO_BIOS:
02487de7 1972 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1973 break;
1974 case LIGHTREC_IO_SCRATCH:
02487de7 1975 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
22eee2ac 1976 break;
ba3814c1
PC		 1977 case LIGHTREC_IO_DIRECT_HW:
1978 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1979 break;
22eee2ac 1980 case LIGHTREC_IO_DIRECT:
02487de7 1981 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
22eee2ac
PC		 1982 break;
1983 default:
98fa08a5 1984 rec_io(state, block, offset, false, true);
9259d748
PC		 1985 return;
1986 }
1987
1988 if (op->i.op == OP_LWC2) {
cb72ea13
PC		 1989 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP);
1990 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
22eee2ac 1991 }
d16005f8
PC		 1992}
1993
98fa08a5 1994static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 1995{
1996 _jit_name(block->_jit, __func__);
02487de7 1997 rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
d16005f8
PC		 1998}
1999
98fa08a5 2000static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2001{
2002 _jit_name(block->_jit, __func__);
02487de7 2003 rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
d16005f8
PC		 2004}
2005
98fa08a5 2006static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2007{
ba3814c1
PC		 2008 jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
2009
d16005f8 2010 _jit_name(block->_jit, __func__);
ba3814c1 2011 rec_load(state, block, offset, code, jit_code_bswapr_us, false);
d16005f8
PC		 2012}
2013
98fa08a5 2014static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2015{
2016 _jit_name(block->_jit, __func__);
02487de7 2017 rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
d16005f8
PC		 2018}
2019
98fa08a5 2020static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2021{
2022 _jit_name(block->_jit, __func__);
98fa08a5 2023 rec_io(state, block, offset, true, true);
d16005f8
PC		 2024}
2025
98fa08a5 2026static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8
PC		 2027{
2028 _jit_name(block->_jit, __func__);
98fa08a5 2029 rec_io(state, block, offset, true, true);
d16005f8
PC		 2030}
2031
98fa08a5 2032static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2033{
9259d748 2034 union code c = block->opcode_list[offset].c;
ba3814c1
PC		 2035 jit_code_t code;
2036
2037 if (is_big_endian() && __WORDSIZE == 64)
2038 code = jit_code_ldxi_ui;
2039 else
2040 code = jit_code_ldxi_i;
2041
9259d748 2042 _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
ba3814c1 2043 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
d16005f8
PC		 2044}
2045
e26c79a8
PC		 2046static void rec_exit_early(struct lightrec_cstate *state,
2047 const struct block *block, u16 offset,
2048 u32 exit_code, u32 pc)
d16005f8 2049{
ba3814c1
PC		 2050 struct regcache *reg_cache = state->reg_cache;
2051 jit_state_t *_jit = block->_jit;
2052 u8 tmp;
2053
98fa08a5 2054 _jit_note(block->_jit, __FILE__, __LINE__);
d16005f8 2055
ba3814c1
PC		 2056 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2057
2058 jit_movi(tmp, exit_code);
8afce295 2059 jit_stxi_i(lightrec_offset(exit_flags), LIGHTREC_REG_STATE, tmp);
ba3814c1 2060
8afce295 2061 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(target_cycle));
cb72ea13
PC		 2062 jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE);
2063 jit_movi(LIGHTREC_REG_CYCLE, 0);
8afce295
PC		 2064 jit_stxi_i(lightrec_offset(target_cycle), LIGHTREC_REG_STATE, tmp);
2065 jit_stxi_i(lightrec_offset(current_cycle), LIGHTREC_REG_STATE, tmp);
cb72ea13 2066
ba3814c1 2067 lightrec_free_reg(reg_cache, tmp);
d16005f8 2068
e26c79a8 2069 lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true);
d16005f8
PC		 2070}
2071
98fa08a5
PC		 2072static void rec_special_SYSCALL(struct lightrec_cstate *state,
2073 const struct block *block, u16 offset)
d16005f8
PC		 2074{
2075 _jit_name(block->_jit, __func__);
e26c79a8
PC		 2076
2077 /* TODO: the return address should be "pc - 4" if we're a delay slot */
2078 rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL,
2079 get_ds_pc(block, offset, 0));
d16005f8
PC		 2080}
2081
98fa08a5
PC		 2082static void rec_special_BREAK(struct lightrec_cstate *state,
2083 const struct block *block, u16 offset)
d16005f8
PC		 2084{
2085 _jit_name(block->_jit, __func__);
e26c79a8
PC		 2086 rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK,
2087 get_ds_pc(block, offset, 0));
d16005f8
PC		 2088}
2089
fdf33147
PC		 2090static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2091{
2092 struct regcache *reg_cache = state->reg_cache;
2093 union code c = block->opcode_list[offset].c;
2094 jit_state_t *_jit = block->_jit;
2095
2096 jit_note(__FILE__, __LINE__);
9259d748
PC		 2097
2098 if (c.i.op != OP_SWC2)
2099 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
fdf33147
PC		 2100
2101 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
2102}
2103
fd58fa32 2104static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
98fa08a5
PC		 2105{
2106 struct regcache *reg_cache = state->reg_cache;
2107 union code c = block->opcode_list[offset].c;
2108 jit_state_t *_jit = block->_jit;
d16005f8 2109
98fa08a5
PC		 2110 jit_note(__FILE__, __LINE__);
2111 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
2112 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
cb72ea13 2113 lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false);
d16005f8 2114
ba3814c1 2115 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
d16005f8 2116
98fa08a5 2117 if (c.i.op == OP_CP0 &&
03535202 2118 !op_flag_no_ds(block->opcode_list[offset].flags) &&
98fa08a5
PC		 2119 (c.r.rd == 12 || c.r.rd == 13))
2120 lightrec_emit_end_of_block(state, block, offset, -1,
2121 get_ds_pc(block, offset, 1),
2122 0, 0, true);
d16005f8
PC		 2123}
2124
98fa08a5
PC		 2125static void
2126rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2127{
d16005f8 2128 struct regcache *reg_cache = state->reg_cache;
98fa08a5 2129 union code c = block->opcode_list[offset].c;
d16005f8 2130 jit_state_t *_jit = block->_jit;
98fa08a5 2131 u8 rt;
d16005f8
PC		 2132
2133 jit_note(__FILE__, __LINE__);
2134
98fa08a5 2135 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
d16005f8 2136
8afce295 2137 jit_ldxi_i(rt, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[c.r.rd]));
d16005f8 2138
98fa08a5
PC		 2139 lightrec_free_reg(reg_cache, rt);
2140}
d16005f8 2141
cb72ea13
PC		 2142static bool block_uses_icache(const struct lightrec_cstate *state,
2143 const struct block *block)
98fa08a5 2144{
cb72ea13 2145 const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM];
98fa08a5 2146 u32 pc = kunseg(block->pc);
d16005f8 2147
cb72ea13
PC		 2148 if (pc < map->pc || pc >= map->pc + map->length)
2149 return false;
2150
2151 return (block->pc >> 28) < 0xa;
d16005f8
PC		 2152}
2153
98fa08a5
PC		 2154static void
2155rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
d16005f8 2156{
98fa08a5
PC		 2157 struct regcache *reg_cache = state->reg_cache;
2158 const union code c = block->opcode_list[offset].c;
2159 jit_state_t *_jit = block->_jit;
fd58fa32 2160 u8 rt, tmp = 0, tmp2, status;
0cf41071 2161 jit_node_t *to_end;
98fa08a5
PC		 2162
2163 jit_note(__FILE__, __LINE__);
2164
2165 switch(c.r.rd) {
2166 case 1:
2167 case 4:
2168 case 8:
2169 case 14:
2170 case 15:
2171 /* Those registers are read-only */
2172 return;
2173 default:
2174 break;
2175 }
2176
cb72ea13
PC		 2177 if (!block_uses_icache(state, block) && c.r.rd == 12) {
2178 /* If we are not running code from the RAM through kuseg or
2179 * kseg0, handle writes to the Status register in C; as the
2180 * code may toggle bit 16 which isolates the cache. Code
2181 * running from kuseg or kseg0 in RAM cannot do that. */
98fa08a5
PC		 2182 rec_mtc(state, block, offset);
2183 return;
2184 }
2185
2186 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
2187
8afce295
PC		 2188 if (c.r.rd != 13)
2189 jit_stxi_i(lightrec_offset(regs.cp0[c.r.rd]), LIGHTREC_REG_STATE, rt);
98fa08a5
PC		 2190
2191 if (c.r.rd == 12 || c.r.rd == 13) {
2192 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
8afce295 2193 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[13]));
fd58fa32
PC		 2194
2195 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
98fa08a5
PC		 2196 }
2197
2198 if (c.r.rd == 12) {
2199 status = rt;
2200 } else if (c.r.rd == 13) {
98fa08a5
PC		 2201 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
2202 jit_andi(tmp2, rt, 0x0300);
2203 jit_ori(tmp, tmp, 0x0300);
2204 jit_xori(tmp, tmp, 0x0300);
2205 jit_orr(tmp, tmp, tmp2);
8afce295
PC		 2206 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[12]));
2207 jit_stxi_i(lightrec_offset(regs.cp0[13]), LIGHTREC_REG_STATE, tmp);
98fa08a5
PC		 2208 status = tmp2;
2209 }
2210
2211 if (c.r.rd == 12 || c.r.rd == 13) {
2212 /* Exit dynarec in case there's a software interrupt.
2213 * exit_flags = !!(status & tmp & 0x0300) & status; */
2214 jit_andr(tmp, tmp, status);
2215 jit_andi(tmp, tmp, 0x0300);
2216 jit_nei(tmp, tmp, 0);
2217 jit_andr(tmp, tmp, status);
fd58fa32
PC		 2218 }
2219
2220 if (c.r.rd == 12) {
2221 /* Exit dynarec in case we unmask a hardware interrupt.
2222 * exit_flags = !(~status & 0x401) */
2223
2224 jit_comr(tmp2, status);
2225 jit_andi(tmp2, tmp2, 0x401);
2226 jit_eqi(tmp2, tmp2, 0);
2227 jit_orr(tmp, tmp, tmp2);
2228 }
2229
0cf41071
PC		 2230 lightrec_free_reg(reg_cache, rt);
2231
fd58fa32 2232 if (c.r.rd == 12 || c.r.rd == 13) {
0cf41071 2233 to_end = jit_beqi(tmp, 0);
98fa08a5 2234
8afce295 2235 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, lightrec_offset(target_cycle));
0cf41071
PC		 2236 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2237 jit_movi(LIGHTREC_REG_CYCLE, 0);
8afce295
PC		 2238 jit_stxi_i(lightrec_offset(target_cycle), LIGHTREC_REG_STATE, tmp2);
2239 jit_stxi_i(lightrec_offset(current_cycle), LIGHTREC_REG_STATE, tmp2);
98fa08a5 2240
0cf41071
PC		 2241
2242 jit_patch(to_end);
2243 }
98fa08a5 2244
03535202 2245 if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
13b02197 2246 (c.r.rd == 12 || c.r.rd == 13)) {
684432ad 2247 state->cycles += lightrec_cycles_of_opcode(state->state, c);
13b02197
PC		 2248 lightrec_emit_eob(state, block, offset + 1);
2249 }
d16005f8
PC		 2250}
2251
98fa08a5
PC		 2252static void rec_cp0_MFC0(struct lightrec_cstate *state,
2253 const struct block *block, u16 offset)
d16005f8
PC		 2254{
2255 _jit_name(block->_jit, __func__);
98fa08a5 2256 rec_mfc0(state, block, offset);
d16005f8
PC		 2257}
2258
98fa08a5
PC		 2259static void rec_cp0_CFC0(struct lightrec_cstate *state,
2260 const struct block *block, u16 offset)
d16005f8
PC		 2261{
2262 _jit_name(block->_jit, __func__);
98fa08a5 2263 rec_mfc0(state, block, offset);
d16005f8
PC		 2264}
2265
98fa08a5
PC		 2266static void rec_cp0_MTC0(struct lightrec_cstate *state,
2267 const struct block *block, u16 offset)
d16005f8
PC		 2268{
2269 _jit_name(block->_jit, __func__);
98fa08a5 2270 rec_mtc0(state, block, offset);
d16005f8
PC		 2271}
2272
98fa08a5
PC		 2273static void rec_cp0_CTC0(struct lightrec_cstate *state,
2274 const struct block *block, u16 offset)
d16005f8
PC		 2275{
2276 _jit_name(block->_jit, __func__);
98fa08a5 2277 rec_mtc0(state, block, offset);
d16005f8
PC		 2278}
2279
02487de7
PC		 2280static unsigned int cp2d_i_offset(u8 reg)
2281{
8afce295 2282 return lightrec_offset(regs.cp2d[reg]);
02487de7
PC		 2283}
2284
2285static unsigned int cp2d_s_offset(u8 reg)
2286{
2287 return cp2d_i_offset(reg) + is_big_endian() * 2;
2288}
2289
2290static unsigned int cp2c_i_offset(u8 reg)
2291{
8afce295 2292 return lightrec_offset(regs.cp2c[reg]);
02487de7
PC		 2293}
2294
2295static unsigned int cp2c_s_offset(u8 reg)
2296{
2297 return cp2c_i_offset(reg) + is_big_endian() * 2;
2298}
2299
9259d748
PC		 2300static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2301 const struct block *block, u16 offset,
2302 u8 reg, u8 out_reg)
d16005f8 2303{
fd58fa32 2304 struct regcache *reg_cache = state->reg_cache;
fd58fa32
PC		 2305 jit_state_t *_jit = block->_jit;
2306 const u32 zext_regs = 0x300f0080;
2307 u8 rt, tmp, tmp2, tmp3, out, flags;
fd58fa32
PC		 2308 unsigned int i;
2309
d16005f8 2310 _jit_name(block->_jit, __func__);
fd58fa32 2311
fdf33147
PC		 2312 if (state->state->ops.cop2_notify) {
2313 /* We must call cop2_notify, handle that in C. */
2314 rec_mfc(state, block, offset);
2315 return;
2316 }
2317
fd58fa32 2318 flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
9259d748
PC		 2319 rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2320
2321 if (reg == 15)
2322 reg = 14;
fd58fa32
PC		 2323
2324 switch (reg) {
2325 case 1:
2326 case 3:
2327 case 5:
2328 case 8:
2329 case 9:
2330 case 10:
2331 case 11:
02487de7 2332 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
fd58fa32
PC		 2333 break;
2334 case 7:
2335 case 16:
2336 case 17:
2337 case 18:
2338 case 19:
02487de7 2339 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
fd58fa32
PC		 2340 break;
2341 case 28:
2342 case 29:
2343 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2344 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2345 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2346
2347 for (i = 0; i < 3; i++) {
2348 out = i == 0 ? rt : tmp;
2349
02487de7 2350 jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
fd58fa32
PC		 2351 jit_movi(tmp2, 0x1f);
2352 jit_rshi(out, tmp, 7);
2353
2354 jit_ltr(tmp3, tmp2, out);
2355 jit_movnr(out, tmp2, tmp3);
2356
2357 jit_gei(tmp2, out, 0);
2358 jit_movzr(out, tmp2, tmp2);
2359
2360 if (i > 0) {
2361 jit_lshi(tmp, tmp, 5 * i);
2362 jit_orr(rt, rt, tmp);
2363 }
2364 }
2365
2366
2367 lightrec_free_reg(reg_cache, tmp);
2368 lightrec_free_reg(reg_cache, tmp2);
2369 lightrec_free_reg(reg_cache, tmp3);
2370 break;
2371 default:
02487de7 2372 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
fd58fa32
PC		 2373 break;
2374 }
2375
2376 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2377}
2378
9259d748
PC		 2379static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2380 const struct block *block, u16 offset)
2381{
2382 const union code c = block->opcode_list[offset].c;
2383
2384 rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2385}
2386
98fa08a5
PC		 2387static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2388 const struct block *block, u16 offset)
d16005f8 2389{
fd58fa32
PC		 2390 struct regcache *reg_cache = state->reg_cache;
2391 const union code c = block->opcode_list[offset].c;
2392 jit_state_t *_jit = block->_jit;
2393 u8 rt;
2394
d16005f8 2395 _jit_name(block->_jit, __func__);
fd58fa32 2396
fdf33147
PC		 2397 if (state->state->ops.cop2_notify) {
2398 /* We must call cop2_notify, handle that in C. */
2399 rec_mfc(state, block, offset);
2400 return;
2401 }
2402
fd58fa32
PC		 2403 switch (c.r.rd) {
2404 case 4:
2405 case 12:
2406 case 20:
2407 case 26:
2408 case 27:
2409 case 29:
2410 case 30:
2411 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
02487de7 2412 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
fd58fa32
PC		 2413 break;
2414 default:
2415 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
0cf41071 2416 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
fd58fa32
PC		 2417 break;
2418 }
2419
2420 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2421}
2422
9259d748
PC		 2423static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2424 const struct block *block, u16 offset,
2425 u8 reg, u8 in_reg)
d16005f8 2426{
fd58fa32 2427 struct regcache *reg_cache = state->reg_cache;
fd58fa32 2428 jit_state_t *_jit = block->_jit;
fd58fa32
PC		 2429 u8 rt, tmp, tmp2, flags = 0;
2430
d16005f8 2431 _jit_name(block->_jit, __func__);
fd58fa32 2432
fdf33147
PC		 2433 if (state->state->ops.cop2_notify) {
2434 /* We must call cop2_notify, handle that in C. */
2435 rec_mtc(state, block, offset);
2436 return;
2437 }
2438
9259d748 2439 if (reg == 31)
fd58fa32
PC		 2440 return;
2441
9259d748 2442 if (reg == 30)
fd58fa32
PC		 2443 flags |= REG_EXT;
2444
9259d748 2445 rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
fd58fa32 2446
9259d748 2447 switch (reg) {
fd58fa32
PC		 2448 case 15:
2449 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
02487de7 2450 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
fd58fa32
PC		 2451
2452 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
02487de7 2453 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
fd58fa32 2454
02487de7
PC		 2455 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2456 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2457 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2458
2459 lightrec_free_reg(reg_cache, tmp);
2460 lightrec_free_reg(reg_cache, tmp2);
2461 break;
2462 case 28:
2463 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2464
2465 jit_lshi(tmp, rt, 7);
2466 jit_andi(tmp, tmp, 0xf80);
02487de7 2467 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2468
2469 jit_lshi(tmp, rt, 2);
2470 jit_andi(tmp, tmp, 0xf80);
02487de7 2471 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2472
2473 jit_rshi(tmp, rt, 3);
2474 jit_andi(tmp, tmp, 0xf80);
02487de7 2475 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2476
2477 lightrec_free_reg(reg_cache, tmp);
2478 break;
2479 case 30:
2480 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
fd58fa32
PC		 2481
2482 /* if (rt < 0) rt = ~rt; */
2483 jit_rshi(tmp, rt, 31);
2484 jit_xorr(tmp, rt, tmp);
2485
cb72ea13
PC		 2486 /* Count leading zeros */
2487 jit_clzr(tmp, tmp);
2488 if (__WORDSIZE != 32)
2489 jit_subi(tmp, tmp, __WORDSIZE - 32);
fd58fa32 2490
cb72ea13 2491 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2492
2493 lightrec_free_reg(reg_cache, tmp);
cb72ea13 2494 fallthrough;
fd58fa32 2495 default:
9259d748 2496 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2497 break;
2498 }
2499
2500 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2501}
2502
9259d748
PC		 2503static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2504 const struct block *block, u16 offset)
2505{
2506 const union code c = block->opcode_list[offset].c;
2507
2508 rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2509}
2510
98fa08a5
PC		 2511static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2512 const struct block *block, u16 offset)
d16005f8 2513{
fd58fa32
PC		 2514 struct regcache *reg_cache = state->reg_cache;
2515 const union code c = block->opcode_list[offset].c;
2516 jit_state_t *_jit = block->_jit;
2517 u8 rt, tmp, tmp2;
2518
98fa08a5 2519 _jit_name(block->_jit, __func__);
fd58fa32 2520
fdf33147
PC		 2521 if (state->state->ops.cop2_notify) {
2522 /* We must call cop2_notify, handle that in C. */
2523 rec_mtc(state, block, offset);
2524 return;
2525 }
2526
fd58fa32
PC		 2527 rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2528
2529 switch (c.r.rd) {
2530 case 4:
2531 case 12:
2532 case 20:
2533 case 26:
2534 case 27:
2535 case 29:
2536 case 30:
02487de7 2537 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2538 break;
2539 case 31:
2540 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2541 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2542
2543 jit_andi(tmp, rt, 0x7f87e000);
2544 jit_nei(tmp, tmp, 0);
2545 jit_lshi(tmp, tmp, 31);
2546
2547 jit_andi(tmp2, rt, 0x7ffff000);
2548 jit_orr(tmp, tmp2, tmp);
2549
02487de7 2550 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
fd58fa32
PC		 2551
2552 lightrec_free_reg(reg_cache, tmp);
2553 lightrec_free_reg(reg_cache, tmp2);
2554 break;
2555
2556 default:
02487de7 2557 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
fd58fa32
PC		 2558 }
2559
2560 lightrec_free_reg(reg_cache, rt);
d16005f8
PC		 2561}
2562
98fa08a5
PC		 2563static void rec_cp0_RFE(struct lightrec_cstate *state,
2564 const struct block *block, u16 offset)
d16005f8 2565{
98fa08a5 2566 struct regcache *reg_cache = state->reg_cache;
d16005f8 2567 jit_state_t *_jit = block->_jit;
98fa08a5 2568 u8 status, tmp;
d16005f8
PC		 2569
2570 jit_name(__func__);
2571 jit_note(__FILE__, __LINE__);
2572
98fa08a5 2573 status = lightrec_alloc_reg_temp(reg_cache, _jit);
8afce295 2574 jit_ldxi_i(status, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[12]));
d16005f8 2575
98fa08a5 2576 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
d16005f8 2577
98fa08a5
PC		 2578 /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2579 jit_rshi(tmp, status, 2);
2580 jit_andi(tmp, tmp, 0xf);
2581 jit_andi(status, status, ~0xful);
2582 jit_orr(status, status, tmp);
2583
8afce295
PC		 2584 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[13]));
2585 jit_stxi_i(lightrec_offset(regs.cp0[12]), LIGHTREC_REG_STATE, status);
98fa08a5
PC		 2586
2587 /* Exit dynarec in case there's a software interrupt.
2588 * exit_flags = !!(status & cause & 0x0300) & status; */
2589 jit_andr(tmp, tmp, status);
2590 jit_andi(tmp, tmp, 0x0300);
2591 jit_nei(tmp, tmp, 0);
2592 jit_andr(tmp, tmp, status);
8afce295 2593 jit_stxi_i(lightrec_offset(exit_flags), LIGHTREC_REG_STATE, tmp);
98fa08a5
PC		 2594
2595 lightrec_free_reg(reg_cache, status);
d16005f8 2596 lightrec_free_reg(reg_cache, tmp);
d16005f8
PC		 2597}
2598
98fa08a5
PC		 2599static void rec_CP(struct lightrec_cstate *state,
2600 const struct block *block, u16 offset)
d16005f8 2601{
98fa08a5 2602 union code c = block->opcode_list[offset].c;
d16005f8
PC		 2603 jit_state_t *_jit = block->_jit;
2604
2605 jit_name(__func__);
2606 jit_note(__FILE__, __LINE__);
2607
ba3814c1 2608 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
d16005f8
PC		 2609}
2610
98fa08a5
PC		 2611static void rec_meta_MOV(struct lightrec_cstate *state,
2612 const struct block *block, u16 offset)
d16005f8 2613{
d16005f8 2614 struct regcache *reg_cache = state->reg_cache;
9259d748
PC		 2615 const struct opcode *op = &block->opcode_list[offset];
2616 union code c = op->c;
d16005f8 2617 jit_state_t *_jit = block->_jit;
9259d748 2618 bool unload_rd;
684432ad 2619 bool unload_rs, discard_rs;
d16005f8
PC		 2620 u8 rs, rd;
2621
2622 _jit_name(block->_jit, __func__);
2623 jit_note(__FILE__, __LINE__);
9259d748 2624
684432ad
PC		 2625 unload_rs = OPT_EARLY_UNLOAD
2626 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD;
2627 discard_rs = OPT_EARLY_UNLOAD
2628 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD;
2629
2630 if ((unload_rs || discard_rs) && c.m.rs) {
2631 /* If the source register is going to be unloaded or discarded,
2632 * then we can simply mark its host register as now pointing to
2633 * the destination register. */
2634 pr_debug("Remap %s to %s at offset 0x%x\n",
2635 lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd),
2636 offset << 2);
2637 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2638 lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs);
2639 lightrec_free_reg(reg_cache, rs);
2640 return;
2641 }
2642
9259d748
PC		 2643 unload_rd = OPT_EARLY_UNLOAD
2644 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2645
8afce295 2646 if (unload_rd) {
9259d748
PC		 2647 /* If the destination register will be unloaded right after the
2648 * MOV meta-opcode, we don't actually need to write any host
2649 * register - we can just store the source register directly to
2650 * the register cache, at the offset corresponding to the
2651 * destination register. */
cb72ea13
PC		 2652 lightrec_discard_reg_if_loaded(reg_cache, c.m.rd);
2653
2654 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
9259d748 2655
8afce295 2656 jit_stxi_i(lightrec_offset(regs.gpr) + (c.m.rd << 2), LIGHTREC_REG_STATE, rs);
d16005f8 2657
98fa08a5 2658 lightrec_free_reg(reg_cache, rs);
9259d748 2659 } else {
cb72ea13
PC		 2660 if (c.m.rs)
2661 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2662
2663 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
9259d748 2664
cb72ea13 2665 if (c.m.rs == 0) {
9259d748 2666 jit_movi(rd, 0);
cb72ea13 2667 } else {
9259d748 2668 jit_extr_i(rd, rs);
cb72ea13
PC		 2669 lightrec_free_reg(reg_cache, rs);
2670 }
9259d748
PC		 2671
2672 lightrec_free_reg(reg_cache, rd);
2673 }
d16005f8
PC		 2674}
2675
98fa08a5
PC		 2676static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2677 const struct block *block,
2678 u16 offset)
d16005f8 2679{
98fa08a5
PC		 2680 struct regcache *reg_cache = state->reg_cache;
2681 union code c = block->opcode_list[offset].c;
d16005f8 2682 jit_state_t *_jit = block->_jit;
cb72ea13 2683 u8 rs, rd;
d16005f8 2684
98fa08a5 2685 _jit_name(block->_jit, __func__);
d16005f8
PC		 2686 jit_note(__FILE__, __LINE__);
2687
684432ad
PC		 2688 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2689 c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd);
d16005f8 2690
cb72ea13
PC		 2691 if (c.m.op == OP_META_EXTC)
2692 jit_extr_c(rd, rs);
98fa08a5 2693 else
cb72ea13 2694 jit_extr_s(rd, rs);
d16005f8 2695
98fa08a5 2696 lightrec_free_reg(reg_cache, rs);
cb72ea13 2697 lightrec_free_reg(reg_cache, rd);
d16005f8
PC		 2698}
2699
ba3814c1
PC		 2700static void rec_meta_MULT2(struct lightrec_cstate *state,
2701 const struct block *block,
2702 u16 offset)
2703{
2704 struct regcache *reg_cache = state->reg_cache;
2705 union code c = block->opcode_list[offset].c;
2706 jit_state_t *_jit = block->_jit;
2707 u8 reg_lo = get_mult_div_lo(c);
2708 u8 reg_hi = get_mult_div_hi(c);
2709 u32 flags = block->opcode_list[offset].flags;
2710 bool is_signed = c.i.op == OP_META_MULT2;
2711 u8 rs, lo, hi, rflags = 0, hiflags = 0;
0cf41071 2712 unsigned int i;
ba3814c1
PC		 2713
2714 if (!op_flag_no_hi(flags) && c.r.op < 32) {
2715 rflags = is_signed ? REG_EXT : REG_ZEXT;
2716 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2717 }
2718
2719 _jit_name(block->_jit, __func__);
2720 jit_note(__FILE__, __LINE__);
2721
2722 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2723
0cf41071
PC		 2724 /*
2725 * We must handle the case where one of the output registers is our rs
2726 * input register. Thanksfully, computing LO/HI can be done in any
2727 * order. Here, we make sure that the computation that overwrites the
2728 * input register is always performed last.
2729 */
2730 for (i = 0; i < 2; i++) {
2731 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2732 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2733
2734 if (c.r.op < 32)
2735 jit_lshi(lo, rs, c.r.op);
2736 else
2737 jit_movi(lo, 0);
ba3814c1 2738
0cf41071
PC		 2739 lightrec_free_reg(reg_cache, lo);
2740 continue;
2741 }
ba3814c1 2742
0cf41071
PC		 2743 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2744 hi = lightrec_alloc_reg_out(reg_cache, _jit,
2745 reg_hi, hiflags);
ba3814c1 2746
f5ee77ca 2747 if (c.r.op >= 32) {
0cf41071 2748 jit_lshi(hi, rs, c.r.op - 32);
f5ee77ca
PC		 2749 } else if (is_signed) {
2750 if (c.r.op)
2751 jit_rshi(hi, rs, 32 - c.r.op);
2752 else
2753 jit_rshi(hi, rs, 31);
2754 } else {
2755 if (c.r.op)
2756 jit_rshi_u(hi, rs, 32 - c.r.op);
2757 else
2758 jit_movi(hi, 0);
2759 }
ba3814c1 2760
0cf41071
PC		 2761 lightrec_free_reg(reg_cache, hi);
2762 }
ba3814c1
PC		 2763 }
2764
2765 lightrec_free_reg(reg_cache, rs);
2766
2767 _jit_name(block->_jit, __func__);
2768 jit_note(__FILE__, __LINE__);
2769}
2770
cb72ea13
PC		 2771static void rec_meta_COM(struct lightrec_cstate *state,
2772 const struct block *block, u16 offset)
2773{
2774 struct regcache *reg_cache = state->reg_cache;
2775 union code c = block->opcode_list[offset].c;
2776 jit_state_t *_jit = block->_jit;
2777 u8 rd, rs, flags;
2778
2779 jit_note(__FILE__, __LINE__);
684432ad
PC		 2780
2781 rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2782 c.m.rs, c.m.rd, 0, 0, &rs, &rd);
cb72ea13
PC		 2783
2784 flags = lightrec_get_reg_in_flags(reg_cache, rs);
2785
2786 lightrec_set_reg_out_flags(reg_cache, rd,
2787 flags & REG_EXT);
2788
2789 jit_comr(rd, rs);
2790
2791 lightrec_free_reg(reg_cache, rs);
2792 lightrec_free_reg(reg_cache, rd);
2793}
2794
5459088b
PC		 2795static void rec_meta_LWU(struct lightrec_cstate *state,
2796 const struct block *block,
2797 u16 offset)
2798{
2799 jit_code_t code;
2800
2801 if (is_big_endian() && __WORDSIZE == 64)
2802 code = jit_code_unldr_u;
2803 else
2804 code = jit_code_unldr;
2805
2806 _jit_name(block->_jit, __func__);
2807 rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2808}
2809
2810static void rec_meta_SWU(struct lightrec_cstate *state,
2811 const struct block *block,
2812 u16 offset)
2813{
2814 _jit_name(block->_jit, __func__);
2815 rec_store(state, block, offset, jit_code_unstr, jit_code_bswapr_ui);
2816}
2817
e26c79a8
PC		 2818static void unknown_opcode(struct lightrec_cstate *state,
2819 const struct block *block, u16 offset)
2820{
2821 rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP,
2822 block->pc + (offset << 2));
2823}
2824
d16005f8 2825static const lightrec_rec_func_t rec_standard[64] = {
98fa08a5 2826 SET_DEFAULT_ELM(rec_standard, unknown_opcode),
d16005f8
PC		 2827 [OP_SPECIAL] = rec_SPECIAL,
2828 [OP_REGIMM] = rec_REGIMM,
2829 [OP_J] = rec_J,
2830 [OP_JAL] = rec_JAL,
2831 [OP_BEQ] = rec_BEQ,
2832 [OP_BNE] = rec_BNE,
2833 [OP_BLEZ] = rec_BLEZ,
2834 [OP_BGTZ] = rec_BGTZ,
2835 [OP_ADDI] = rec_ADDI,
2836 [OP_ADDIU] = rec_ADDIU,
2837 [OP_SLTI] = rec_SLTI,
2838 [OP_SLTIU] = rec_SLTIU,
2839 [OP_ANDI] = rec_ANDI,
2840 [OP_ORI] = rec_ORI,
2841 [OP_XORI] = rec_XORI,
2842 [OP_LUI] = rec_LUI,
2843 [OP_CP0] = rec_CP0,
2844 [OP_CP2] = rec_CP2,
2845 [OP_LB] = rec_LB,
2846 [OP_LH] = rec_LH,
2847 [OP_LWL] = rec_LWL,
2848 [OP_LW] = rec_LW,
2849 [OP_LBU] = rec_LBU,
2850 [OP_LHU] = rec_LHU,
2851 [OP_LWR] = rec_LWR,
2852 [OP_SB] = rec_SB,
2853 [OP_SH] = rec_SH,
2854 [OP_SWL] = rec_SWL,
2855 [OP_SW] = rec_SW,
2856 [OP_SWR] = rec_SWR,
9259d748
PC		 2857 [OP_LWC2] = rec_LW,
2858 [OP_SWC2] = rec_SW,
d16005f8 2859
cb72ea13 2860 [OP_META] = rec_META,
ba3814c1
PC		 2861 [OP_META_MULT2] = rec_meta_MULT2,
2862 [OP_META_MULTU2] = rec_meta_MULT2,
5459088b
PC		 2863 [OP_META_LWU] = rec_meta_LWU,
2864 [OP_META_SWU] = rec_meta_SWU,
d16005f8
PC		 2865};
2866
2867static const lightrec_rec_func_t rec_special[64] = {
98fa08a5 2868 SET_DEFAULT_ELM(rec_special, unknown_opcode),
d16005f8
PC		 2869 [OP_SPECIAL_SLL] = rec_special_SLL,
2870 [OP_SPECIAL_SRL] = rec_special_SRL,
2871 [OP_SPECIAL_SRA] = rec_special_SRA,
2872 [OP_SPECIAL_SLLV] = rec_special_SLLV,
2873 [OP_SPECIAL_SRLV] = rec_special_SRLV,
2874 [OP_SPECIAL_SRAV] = rec_special_SRAV,
2875 [OP_SPECIAL_JR] = rec_special_JR,
2876 [OP_SPECIAL_JALR] = rec_special_JALR,
2877 [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL,
2878 [OP_SPECIAL_BREAK] = rec_special_BREAK,
2879 [OP_SPECIAL_MFHI] = rec_special_MFHI,
2880 [OP_SPECIAL_MTHI] = rec_special_MTHI,
2881 [OP_SPECIAL_MFLO] = rec_special_MFLO,
2882 [OP_SPECIAL_MTLO] = rec_special_MTLO,
2883 [OP_SPECIAL_MULT] = rec_special_MULT,
2884 [OP_SPECIAL_MULTU] = rec_special_MULTU,
2885 [OP_SPECIAL_DIV] = rec_special_DIV,
2886 [OP_SPECIAL_DIVU] = rec_special_DIVU,
2887 [OP_SPECIAL_ADD] = rec_special_ADD,
2888 [OP_SPECIAL_ADDU] = rec_special_ADDU,
2889 [OP_SPECIAL_SUB] = rec_special_SUB,
2890 [OP_SPECIAL_SUBU] = rec_special_SUBU,
2891 [OP_SPECIAL_AND] = rec_special_AND,
2892 [OP_SPECIAL_OR] = rec_special_OR,
2893 [OP_SPECIAL_XOR] = rec_special_XOR,
2894 [OP_SPECIAL_NOR] = rec_special_NOR,
2895 [OP_SPECIAL_SLT] = rec_special_SLT,
2896 [OP_SPECIAL_SLTU] = rec_special_SLTU,
2897};
2898
2899static const lightrec_rec_func_t rec_regimm[64] = {
98fa08a5 2900 SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
d16005f8
PC		 2901 [OP_REGIMM_BLTZ] = rec_regimm_BLTZ,
2902 [OP_REGIMM_BGEZ] = rec_regimm_BGEZ,
2903 [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL,
2904 [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL,
2905};
2906
2907static const lightrec_rec_func_t rec_cp0[64] = {
98fa08a5 2908 SET_DEFAULT_ELM(rec_cp0, rec_CP),
d16005f8
PC		 2909 [OP_CP0_MFC0] = rec_cp0_MFC0,
2910 [OP_CP0_CFC0] = rec_cp0_CFC0,
2911 [OP_CP0_MTC0] = rec_cp0_MTC0,
2912 [OP_CP0_CTC0] = rec_cp0_CTC0,
2913 [OP_CP0_RFE] = rec_cp0_RFE,
2914};
2915
2916static const lightrec_rec_func_t rec_cp2_basic[64] = {
98fa08a5 2917 SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
d16005f8
PC		 2918 [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2,
2919 [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2,
2920 [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2,
2921 [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2,
2922};
2923
cb72ea13
PC		 2924static const lightrec_rec_func_t rec_meta[64] = {
2925 SET_DEFAULT_ELM(rec_meta, unknown_opcode),
2926 [OP_META_MOV] = rec_meta_MOV,
2927 [OP_META_EXTC] = rec_meta_EXTC_EXTS,
2928 [OP_META_EXTS] = rec_meta_EXTC_EXTS,
2929 [OP_META_COM] = rec_meta_COM,
2930};
2931
98fa08a5
PC		 2932static void rec_SPECIAL(struct lightrec_cstate *state,
2933 const struct block *block, u16 offset)
d16005f8 2934{
98fa08a5
PC		 2935 union code c = block->opcode_list[offset].c;
2936 lightrec_rec_func_t f = rec_special[c.r.op];
2937
2938 if (!HAS_DEFAULT_ELM && unlikely(!f))
2939 unknown_opcode(state, block, offset);
d16005f8 2940 else
98fa08a5 2941 (*f)(state, block, offset);
d16005f8
PC		 2942}
2943
98fa08a5
PC		 2944static void rec_REGIMM(struct lightrec_cstate *state,
2945 const struct block *block, u16 offset)
d16005f8 2946{
98fa08a5
PC		 2947 union code c = block->opcode_list[offset].c;
2948 lightrec_rec_func_t f = rec_regimm[c.r.rt];
2949
2950 if (!HAS_DEFAULT_ELM && unlikely(!f))
2951 unknown_opcode(state, block, offset);
d16005f8 2952 else
98fa08a5 2953 (*f)(state, block, offset);
d16005f8
PC		 2954}
2955
98fa08a5
PC		 2956static void rec_CP0(struct lightrec_cstate *state,
2957 const struct block *block, u16 offset)
d16005f8 2958{
98fa08a5
PC		 2959 union code c = block->opcode_list[offset].c;
2960 lightrec_rec_func_t f = rec_cp0[c.r.rs];
2961
2962 if (!HAS_DEFAULT_ELM && unlikely(!f))
2963 rec_CP(state, block, offset);
d16005f8 2964 else
98fa08a5 2965 (*f)(state, block, offset);
d16005f8
PC		 2966}
2967
98fa08a5
PC		 2968static void rec_CP2(struct lightrec_cstate *state,
2969 const struct block *block, u16 offset)
d16005f8 2970{
98fa08a5
PC		 2971 union code c = block->opcode_list[offset].c;
2972
2973 if (c.r.op == OP_CP2_BASIC) {
2974 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2975
2976 if (HAS_DEFAULT_ELM || likely(f)) {
2977 (*f)(state, block, offset);
d16005f8
PC		 2978 return;
2979 }
2980 }
2981
98fa08a5 2982 rec_CP(state, block, offset);
d16005f8
PC		 2983}
2984
cb72ea13
PC		 2985static void rec_META(struct lightrec_cstate *state,
2986 const struct block *block, u16 offset)
2987{
2988 union code c = block->opcode_list[offset].c;
2989 lightrec_rec_func_t f = rec_meta[c.m.op];
2990
2991 if (!HAS_DEFAULT_ELM && unlikely(!f))
2992 unknown_opcode(state, block, offset);
2993 else
2994 (*f)(state, block, offset);
2995}
2996
98fa08a5
PC		 2997void lightrec_rec_opcode(struct lightrec_cstate *state,
2998 const struct block *block, u16 offset)
d16005f8 2999{
98fa08a5
PC		 3000 struct regcache *reg_cache = state->reg_cache;
3001 struct lightrec_branch_target *target;
3002 const struct opcode *op = &block->opcode_list[offset];
3003 jit_state_t *_jit = block->_jit;
3004 lightrec_rec_func_t f;
03535202 3005 u16 unload_offset;
98fa08a5 3006
03535202
PC		 3007 if (op_flag_sync(op->flags)) {
3008 if (state->cycles)
3009 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
98fa08a5
PC		 3010 state->cycles = 0;
3011
3012 lightrec_storeback_regs(reg_cache, _jit);
3013 lightrec_regcache_reset(reg_cache);
3014
3015 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
3016 target = &state->targets[state->nb_targets++];
3017 target->offset = offset;
3018 target->label = jit_indirect();
3019 }
3020
3021 if (likely(op->opcode)) {
3022 f = rec_standard[op->i.op];
3023
3024 if (!HAS_DEFAULT_ELM && unlikely(!f))
3025 unknown_opcode(state, block, offset);
3026 else
3027 (*f)(state, block, offset);
3028 }
3029
03535202
PC		 3030 if (OPT_EARLY_UNLOAD) {
3031 unload_offset = offset +
3032 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
3033
3034 lightrec_do_early_unload(state, block, unload_offset);
98fa08a5 3035 }
cb72ea13
PC		 3036
3037 state->no_load_delay = false;
d16005f8 3038}
ARM optimized PCSX fork