Commit | Line | Data |
---|---|---|
98fa08a5 | 1 | // SPDX-License-Identifier: LGPL-2.1-or-later |
d16005f8 | 2 | /* |
98fa08a5 | 3 | * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net> |
d16005f8 PC |
4 | */ |
5 | ||
6 | #include "blockcache.h" | |
7 | #include "debug.h" | |
8 | #include "disassembler.h" | |
9 | #include "emitter.h" | |
98fa08a5 | 10 | #include "lightning-wrapper.h" |
d16005f8 PC |
11 | #include "optimizer.h" |
12 | #include "regcache.h" | |
13 | ||
d16005f8 PC |
14 | #include <stdbool.h> |
15 | #include <stddef.h> | |
16 | ||
98fa08a5 | 17 | typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16); |
d16005f8 PC |
18 | |
19 | /* Forward declarations */ | |
98fa08a5 PC |
20 | static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset); |
21 | static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
22 | static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
23 | static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
cb72ea13 | 24 | static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset); |
9259d748 PC |
25 | static void rec_cp2_do_mtc2(struct lightrec_cstate *state, |
26 | const struct block *block, u16 offset, u8 reg, u8 in_reg); | |
27 | static void rec_cp2_do_mfc2(struct lightrec_cstate *state, | |
28 | const struct block *block, u16 offset, | |
29 | u8 reg, u8 out_reg); | |
d16005f8 | 30 | |
ba3814c1 | 31 | static void |
cb72ea13 | 32 | lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void)) |
ba3814c1 PC |
33 | { |
34 | /* Prevent jit_jmpi() from using our cycles register as a temporary */ | |
35 | jit_live(LIGHTREC_REG_CYCLE); | |
36 | ||
cb72ea13 PC |
37 | jit_patch_abs(jit_jmpi(), fn); |
38 | } | |
39 | ||
40 | static void | |
41 | lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit) | |
42 | { | |
43 | lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func); | |
44 | } | |
45 | ||
46 | static void | |
47 | lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit) | |
48 | { | |
49 | lightrec_jump_to_fn(_jit, state->state->ds_check_func); | |
ba3814c1 PC |
50 | } |
51 | ||
9259d748 PC |
52 | static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit, |
53 | u8 ra_reg, u32 pc, u32 link) | |
54 | { | |
55 | u8 link_reg; | |
56 | ||
57 | link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0); | |
58 | lightrec_load_imm(reg_cache, _jit, link_reg, pc, link); | |
59 | lightrec_free_reg(reg_cache, link_reg); | |
60 | } | |
61 | ||
98fa08a5 PC |
62 | static void lightrec_emit_end_of_block(struct lightrec_cstate *state, |
63 | const struct block *block, u16 offset, | |
d16005f8 PC |
64 | s8 reg_new_pc, u32 imm, u8 ra_reg, |
65 | u32 link, bool update_cycles) | |
66 | { | |
d16005f8 | 67 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 68 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 69 | const struct opcode *op = &block->opcode_list[offset], |
cb72ea13 | 70 | *ds = get_delay_slot(block->opcode_list, offset); |
684432ad | 71 | u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c); |
d16005f8 PC |
72 | |
73 | jit_note(__FILE__, __LINE__); | |
74 | ||
9259d748 PC |
75 | if (link && ra_reg != reg_new_pc) |
76 | update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); | |
d16005f8 | 77 | |
9259d748 PC |
78 | if (reg_new_pc < 0) |
79 | lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm); | |
80 | else | |
81 | lightrec_load_next_pc(reg_cache, _jit, reg_new_pc); | |
d16005f8 | 82 | |
9259d748 PC |
83 | if (link && ra_reg == reg_new_pc) { |
84 | /* Handle the special case: JALR $r0, $r0 | |
85 | * In that case the target PC should be the old value of the | |
86 | * register. */ | |
87 | update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); | |
d16005f8 PC |
88 | } |
89 | ||
90 | if (has_delay_slot(op->c) && | |
03535202 | 91 | !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) { |
684432ad | 92 | cycles += lightrec_cycles_of_opcode(state->state, ds->c); |
d16005f8 PC |
93 | |
94 | /* Recompile the delay slot */ | |
cb72ea13 | 95 | if (ds->c.opcode) |
98fa08a5 | 96 | lightrec_rec_opcode(state, block, offset + 1); |
d16005f8 PC |
97 | } |
98 | ||
03535202 PC |
99 | /* Clean the remaining registers */ |
100 | lightrec_clean_regs(reg_cache, _jit); | |
d16005f8 | 101 | |
d16005f8 PC |
102 | if (cycles && update_cycles) { |
103 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); | |
104 | pr_debug("EOB: %u cycles\n", cycles); | |
105 | } | |
106 | ||
cb72ea13 PC |
107 | if (op_flag_load_delay(ds->flags) |
108 | && opcode_is_load(ds->c) && !state->no_load_delay) { | |
109 | /* If the delay slot is a load opcode, its target register | |
110 | * will be written after the first opcode of the target is | |
111 | * executed. Handle this by jumping to a special section of | |
112 | * the dispatcher. It expects the loaded value to be in | |
113 | * REG_TEMP, and the target register number to be in JIT_V1.*/ | |
114 | jit_movi(JIT_V1, ds->c.i.rt); | |
115 | ||
116 | lightrec_jump_to_ds_check(state, _jit); | |
117 | } else { | |
118 | lightrec_jump_to_eob(state, _jit); | |
119 | } | |
0e720fb1 PC |
120 | |
121 | lightrec_regcache_reset(reg_cache); | |
d16005f8 PC |
122 | } |
123 | ||
cb72ea13 PC |
124 | void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state, |
125 | const struct block *block, u16 offset) | |
126 | { | |
127 | struct regcache *reg_cache = state->reg_cache; | |
128 | jit_state_t *_jit = block->_jit; | |
129 | ||
130 | lightrec_clean_regs(reg_cache, _jit); | |
131 | ||
132 | /* Call the interpreter with the block's address in JIT_V1 and the | |
133 | * PC (which might have an offset) in JIT_V0. */ | |
134 | lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, | |
135 | block->pc + (offset << 2)); | |
0e720fb1 PC |
136 | if (lightrec_store_next_pc()) { |
137 | jit_stxi_i(offsetof(struct lightrec_state, next_pc), | |
138 | LIGHTREC_REG_STATE, JIT_V0); | |
139 | } | |
140 | ||
cb72ea13 PC |
141 | jit_movi(JIT_V1, (uintptr_t)block); |
142 | ||
143 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); | |
144 | lightrec_jump_to_fn(_jit, state->state->interpreter_func); | |
145 | } | |
146 | ||
147 | static void lightrec_emit_eob(struct lightrec_cstate *state, | |
148 | const struct block *block, u16 offset) | |
d16005f8 | 149 | { |
d16005f8 PC |
150 | struct regcache *reg_cache = state->reg_cache; |
151 | jit_state_t *_jit = block->_jit; | |
152 | ||
03535202 | 153 | lightrec_clean_regs(reg_cache, _jit); |
d16005f8 | 154 | |
9259d748 PC |
155 | lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, |
156 | block->pc + (offset << 2)); | |
0e720fb1 PC |
157 | if (lightrec_store_next_pc()) { |
158 | jit_stxi_i(offsetof(struct lightrec_state, next_pc), | |
159 | LIGHTREC_REG_STATE, JIT_V0); | |
160 | } | |
161 | ||
13b02197 | 162 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); |
d16005f8 | 163 | |
ba3814c1 | 164 | lightrec_jump_to_eob(state, _jit); |
d16005f8 PC |
165 | } |
166 | ||
98fa08a5 | 167 | static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 168 | { |
9259d748 | 169 | union code c = block->opcode_list[offset].c; |
d16005f8 PC |
170 | |
171 | _jit_name(block->_jit, __func__); | |
9259d748 | 172 | lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true); |
98fa08a5 PC |
173 | } |
174 | ||
175 | static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
176 | { | |
98fa08a5 PC |
177 | union code c = block->opcode_list[offset].c; |
178 | ||
179 | _jit_name(block->_jit, __func__); | |
9259d748 | 180 | lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd, |
98fa08a5 | 181 | get_branch_pc(block, offset, 2), true); |
d16005f8 PC |
182 | } |
183 | ||
98fa08a5 | 184 | static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 185 | { |
98fa08a5 PC |
186 | union code c = block->opcode_list[offset].c; |
187 | ||
d16005f8 | 188 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
189 | lightrec_emit_end_of_block(state, block, offset, -1, |
190 | (block->pc & 0xf0000000) | (c.j.imm << 2), | |
191 | 31, 0, true); | |
d16005f8 PC |
192 | } |
193 | ||
98fa08a5 | 194 | static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 195 | { |
98fa08a5 PC |
196 | union code c = block->opcode_list[offset].c; |
197 | ||
d16005f8 | 198 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
199 | lightrec_emit_end_of_block(state, block, offset, -1, |
200 | (block->pc & 0xf0000000) | (c.j.imm << 2), | |
201 | 31, get_branch_pc(block, offset, 2), true); | |
d16005f8 PC |
202 | } |
203 | ||
03535202 PC |
204 | static void lightrec_do_early_unload(struct lightrec_cstate *state, |
205 | const struct block *block, u16 offset) | |
206 | { | |
207 | struct regcache *reg_cache = state->reg_cache; | |
208 | const struct opcode *op = &block->opcode_list[offset]; | |
209 | jit_state_t *_jit = block->_jit; | |
210 | unsigned int i; | |
211 | u8 reg; | |
212 | struct { | |
213 | u8 reg, op; | |
214 | } reg_ops[3] = { | |
215 | { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), }, | |
216 | { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), }, | |
217 | { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), }, | |
218 | }; | |
219 | ||
220 | for (i = 0; i < ARRAY_SIZE(reg_ops); i++) { | |
221 | reg = reg_ops[i].reg; | |
222 | ||
223 | switch (reg_ops[i].op) { | |
224 | case LIGHTREC_REG_UNLOAD: | |
225 | lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true); | |
226 | break; | |
227 | ||
228 | case LIGHTREC_REG_DISCARD: | |
229 | lightrec_discard_reg_if_loaded(reg_cache, reg); | |
230 | break; | |
231 | ||
232 | case LIGHTREC_REG_CLEAN: | |
233 | lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false); | |
234 | break; | |
235 | default: | |
236 | break; | |
237 | }; | |
238 | } | |
239 | } | |
240 | ||
98fa08a5 | 241 | static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset, |
ba3814c1 | 242 | jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz) |
d16005f8 | 243 | { |
98fa08a5 | 244 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 PC |
245 | struct native_register *regs_backup; |
246 | jit_state_t *_jit = block->_jit; | |
247 | struct lightrec_branch *branch; | |
98fa08a5 | 248 | const struct opcode *op = &block->opcode_list[offset], |
cb72ea13 | 249 | *ds = get_delay_slot(block->opcode_list, offset); |
d16005f8 | 250 | jit_node_t *addr; |
cb72ea13 | 251 | bool is_forward = (s16)op->i.imm >= 0; |
684432ad | 252 | int op_cycles = lightrec_cycles_of_opcode(state->state, op->c); |
03535202 | 253 | u32 target_offset, cycles = state->cycles + op_cycles; |
ba3814c1 | 254 | bool no_indirection = false; |
98fa08a5 | 255 | u32 next_pc; |
9259d748 | 256 | u8 rs, rt; |
d16005f8 PC |
257 | |
258 | jit_note(__FILE__, __LINE__); | |
259 | ||
03535202 | 260 | if (!op_flag_no_ds(op->flags)) |
684432ad | 261 | cycles += lightrec_cycles_of_opcode(state->state, ds->c); |
d16005f8 | 262 | |
03535202 PC |
263 | state->cycles = -op_cycles; |
264 | ||
265 | if (!unconditional) { | |
266 | rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT); | |
267 | rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache, | |
268 | _jit, op->i.rt, REG_EXT); | |
269 | ||
270 | /* Unload dead registers before evaluating the branch */ | |
271 | if (OPT_EARLY_UNLOAD) | |
272 | lightrec_do_early_unload(state, block, offset); | |
ba3814c1 PC |
273 | |
274 | if (op_flag_local_branch(op->flags) && | |
cb72ea13 | 275 | (op_flag_no_ds(op->flags) || !ds->opcode) && |
ba3814c1 PC |
276 | is_forward && !lightrec_has_dirty_regs(reg_cache)) |
277 | no_indirection = true; | |
278 | ||
279 | if (no_indirection) | |
280 | pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2); | |
03535202 | 281 | } |
d16005f8 PC |
282 | |
283 | if (cycles) | |
284 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); | |
285 | ||
286 | if (!unconditional) { | |
d16005f8 | 287 | /* Generate the branch opcode */ |
ba3814c1 PC |
288 | if (!no_indirection) |
289 | addr = jit_new_node_pww(code, NULL, rs, rt); | |
d16005f8 PC |
290 | |
291 | lightrec_free_regs(reg_cache); | |
292 | regs_backup = lightrec_regcache_enter_branch(reg_cache); | |
293 | } | |
294 | ||
03535202 PC |
295 | if (op_flag_local_branch(op->flags)) { |
296 | /* Recompile the delay slot */ | |
cb72ea13 PC |
297 | if (!op_flag_no_ds(op->flags) && ds->opcode) { |
298 | /* Never handle load delays with local branches. */ | |
299 | state->no_load_delay = true; | |
03535202 | 300 | lightrec_rec_opcode(state, block, offset + 1); |
cb72ea13 | 301 | } |
d16005f8 | 302 | |
9259d748 PC |
303 | if (link) |
304 | update_ra_register(reg_cache, _jit, 31, block->pc, link); | |
d16005f8 | 305 | |
03535202 PC |
306 | /* Clean remaining registers */ |
307 | lightrec_clean_regs(reg_cache, _jit); | |
d16005f8 | 308 | |
98fa08a5 | 309 | target_offset = offset + 1 + (s16)op->i.imm |
03535202 | 310 | - !!op_flag_no_ds(op->flags); |
98fa08a5 PC |
311 | pr_debug("Adding local branch to offset 0x%x\n", |
312 | target_offset << 2); | |
313 | branch = &state->local_branches[ | |
314 | state->nb_local_branches++]; | |
d16005f8 | 315 | |
98fa08a5 | 316 | branch->target = target_offset; |
ba3814c1 PC |
317 | |
318 | if (no_indirection) | |
319 | branch->branch = jit_new_node_pww(code2, NULL, rs, rt); | |
320 | else if (is_forward) | |
22eee2ac | 321 | branch->branch = jit_b(); |
d16005f8 PC |
322 | else |
323 | branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0); | |
324 | } | |
325 | ||
03535202 | 326 | if (!op_flag_local_branch(op->flags) || !is_forward) { |
98fa08a5 | 327 | next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm); |
cb72ea13 | 328 | state->no_load_delay = op_flag_local_branch(op->flags); |
98fa08a5 | 329 | lightrec_emit_end_of_block(state, block, offset, -1, next_pc, |
d16005f8 PC |
330 | 31, link, false); |
331 | } | |
332 | ||
333 | if (!unconditional) { | |
ba3814c1 PC |
334 | if (!no_indirection) |
335 | jit_patch(addr); | |
336 | ||
d16005f8 PC |
337 | lightrec_regcache_leave_branch(reg_cache, regs_backup); |
338 | ||
9259d748 PC |
339 | if (bz && link) |
340 | update_ra_register(reg_cache, _jit, 31, block->pc, link); | |
d16005f8 | 341 | |
cb72ea13 PC |
342 | if (!op_flag_no_ds(op->flags) && ds->opcode) { |
343 | state->no_load_delay = true; | |
98fa08a5 | 344 | lightrec_rec_opcode(state, block, offset + 1); |
cb72ea13 | 345 | } |
d16005f8 PC |
346 | } |
347 | } | |
348 | ||
98fa08a5 PC |
349 | static void rec_BNE(struct lightrec_cstate *state, |
350 | const struct block *block, u16 offset) | |
d16005f8 | 351 | { |
98fa08a5 PC |
352 | union code c = block->opcode_list[offset].c; |
353 | ||
d16005f8 | 354 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
355 | |
356 | if (c.i.rt == 0) | |
ba3814c1 | 357 | rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true); |
98fa08a5 | 358 | else |
ba3814c1 | 359 | rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false); |
d16005f8 PC |
360 | } |
361 | ||
98fa08a5 PC |
362 | static void rec_BEQ(struct lightrec_cstate *state, |
363 | const struct block *block, u16 offset) | |
d16005f8 | 364 | { |
98fa08a5 PC |
365 | union code c = block->opcode_list[offset].c; |
366 | ||
d16005f8 | 367 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
368 | |
369 | if (c.i.rt == 0) | |
ba3814c1 | 370 | rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true); |
98fa08a5 | 371 | else |
ba3814c1 | 372 | rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false); |
d16005f8 PC |
373 | } |
374 | ||
98fa08a5 PC |
375 | static void rec_BLEZ(struct lightrec_cstate *state, |
376 | const struct block *block, u16 offset) | |
d16005f8 | 377 | { |
98fa08a5 PC |
378 | union code c = block->opcode_list[offset].c; |
379 | ||
d16005f8 | 380 | _jit_name(block->_jit, __func__); |
ba3814c1 | 381 | rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true); |
d16005f8 PC |
382 | } |
383 | ||
98fa08a5 PC |
384 | static void rec_BGTZ(struct lightrec_cstate *state, |
385 | const struct block *block, u16 offset) | |
d16005f8 PC |
386 | { |
387 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 388 | rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true); |
d16005f8 PC |
389 | } |
390 | ||
98fa08a5 PC |
391 | static void rec_regimm_BLTZ(struct lightrec_cstate *state, |
392 | const struct block *block, u16 offset) | |
d16005f8 PC |
393 | { |
394 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 395 | rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true); |
d16005f8 PC |
396 | } |
397 | ||
98fa08a5 PC |
398 | static void rec_regimm_BLTZAL(struct lightrec_cstate *state, |
399 | const struct block *block, u16 offset) | |
d16005f8 PC |
400 | { |
401 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 402 | rec_b(state, block, offset, jit_code_bgei, jit_code_blti, |
98fa08a5 | 403 | get_branch_pc(block, offset, 2), false, true); |
d16005f8 PC |
404 | } |
405 | ||
98fa08a5 PC |
406 | static void rec_regimm_BGEZ(struct lightrec_cstate *state, |
407 | const struct block *block, u16 offset) | |
d16005f8 | 408 | { |
98fa08a5 PC |
409 | union code c = block->opcode_list[offset].c; |
410 | ||
d16005f8 | 411 | _jit_name(block->_jit, __func__); |
ba3814c1 | 412 | rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true); |
d16005f8 PC |
413 | } |
414 | ||
98fa08a5 PC |
415 | static void rec_regimm_BGEZAL(struct lightrec_cstate *state, |
416 | const struct block *block, u16 offset) | |
d16005f8 | 417 | { |
98fa08a5 | 418 | const struct opcode *op = &block->opcode_list[offset]; |
d16005f8 | 419 | _jit_name(block->_jit, __func__); |
ba3814c1 | 420 | rec_b(state, block, offset, jit_code_blti, jit_code_bgei, |
98fa08a5 PC |
421 | get_branch_pc(block, offset, 2), |
422 | !op->i.rs, true); | |
d16005f8 PC |
423 | } |
424 | ||
684432ad PC |
425 | static void rec_alloc_rs_rd(struct regcache *reg_cache, |
426 | jit_state_t *_jit, | |
427 | const struct opcode *op, | |
428 | u8 rs, u8 rd, | |
429 | u8 in_flags, u8 out_flags, | |
430 | u8 *rs_out, u8 *rd_out) | |
431 | { | |
432 | bool unload, discard; | |
433 | u32 unload_flags; | |
434 | ||
435 | if (OPT_EARLY_UNLOAD) { | |
436 | unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags); | |
437 | unload = unload_flags == LIGHTREC_REG_UNLOAD; | |
438 | discard = unload_flags == LIGHTREC_REG_DISCARD; | |
439 | } | |
440 | ||
441 | if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) { | |
442 | rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags); | |
443 | lightrec_remap_reg(reg_cache, _jit, rs, rd, discard); | |
444 | lightrec_set_reg_out_flags(reg_cache, rs, out_flags); | |
445 | rd = rs; | |
446 | } else { | |
447 | rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags); | |
448 | rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags); | |
449 | } | |
450 | ||
451 | *rs_out = rs; | |
452 | *rd_out = rd; | |
453 | } | |
454 | ||
98fa08a5 PC |
455 | static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block, |
456 | u16 offset, jit_code_t code, bool slti) | |
d16005f8 | 457 | { |
98fa08a5 PC |
458 | struct regcache *reg_cache = state->reg_cache; |
459 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 460 | jit_state_t *_jit = block->_jit; |
98fa08a5 PC |
461 | u8 rs, rt, out_flags = REG_EXT; |
462 | ||
463 | if (slti) | |
464 | out_flags |= REG_ZEXT; | |
d16005f8 PC |
465 | |
466 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
467 | |
468 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
469 | c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt); | |
d16005f8 | 470 | |
98fa08a5 | 471 | jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm); |
d16005f8 PC |
472 | |
473 | lightrec_free_reg(reg_cache, rs); | |
474 | lightrec_free_reg(reg_cache, rt); | |
475 | } | |
476 | ||
98fa08a5 PC |
477 | static void rec_alu_special(struct lightrec_cstate *state, const struct block *block, |
478 | u16 offset, jit_code_t code, bool out_ext) | |
d16005f8 | 479 | { |
98fa08a5 PC |
480 | struct regcache *reg_cache = state->reg_cache; |
481 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
482 | jit_state_t *_jit = block->_jit; |
483 | u8 rd, rt, rs; | |
484 | ||
485 | jit_note(__FILE__, __LINE__); | |
684432ad | 486 | |
98fa08a5 | 487 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT); |
684432ad PC |
488 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
489 | c.r.rs, c.r.rd, REG_EXT, | |
490 | out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd); | |
d16005f8 PC |
491 | |
492 | jit_new_node_www(code, rd, rs, rt); | |
493 | ||
494 | lightrec_free_reg(reg_cache, rs); | |
495 | lightrec_free_reg(reg_cache, rt); | |
496 | lightrec_free_reg(reg_cache, rd); | |
497 | } | |
498 | ||
98fa08a5 PC |
499 | static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block, |
500 | u16 offset, jit_code_t code) | |
d16005f8 | 501 | { |
98fa08a5 PC |
502 | struct regcache *reg_cache = state->reg_cache; |
503 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 504 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 505 | u8 rd, rt, rs, temp, flags = 0; |
d16005f8 PC |
506 | |
507 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 508 | |
98fa08a5 PC |
509 | if (code == jit_code_rshr) |
510 | flags = REG_EXT; | |
511 | else if (code == jit_code_rshr_u) | |
512 | flags = REG_ZEXT; | |
d16005f8 | 513 | |
684432ad PC |
514 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); |
515 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
516 | c.r.rt, c.r.rd, flags, flags, &rt, &rd); | |
d16005f8 | 517 | |
684432ad | 518 | if (rt != rd) { |
98fa08a5 PC |
519 | jit_andi(rd, rs, 0x1f); |
520 | jit_new_node_www(code, rd, rt, rd); | |
521 | } else { | |
522 | temp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
523 | jit_andi(temp, rs, 0x1f); | |
d16005f8 | 524 | jit_new_node_www(code, rd, rt, temp); |
98fa08a5 PC |
525 | lightrec_free_reg(reg_cache, temp); |
526 | } | |
d16005f8 PC |
527 | |
528 | lightrec_free_reg(reg_cache, rs); | |
d16005f8 PC |
529 | lightrec_free_reg(reg_cache, rt); |
530 | lightrec_free_reg(reg_cache, rd); | |
531 | } | |
532 | ||
02487de7 PC |
533 | static void rec_movi(struct lightrec_cstate *state, |
534 | const struct block *block, u16 offset) | |
535 | { | |
536 | struct regcache *reg_cache = state->reg_cache; | |
537 | union code c = block->opcode_list[offset].c; | |
538 | jit_state_t *_jit = block->_jit; | |
539 | u16 flags = REG_EXT; | |
684432ad | 540 | s32 value = (s32)(s16) c.i.imm; |
02487de7 PC |
541 | u8 rt; |
542 | ||
684432ad PC |
543 | if (block->opcode_list[offset].flags & LIGHTREC_MOVI) |
544 | value += (s32)((u32)state->movi_temp[c.i.rt] << 16); | |
545 | ||
546 | if (value >= 0) | |
02487de7 PC |
547 | flags |= REG_ZEXT; |
548 | ||
549 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); | |
550 | ||
684432ad | 551 | jit_movi(rt, value); |
02487de7 PC |
552 | |
553 | lightrec_free_reg(reg_cache, rt); | |
554 | } | |
555 | ||
98fa08a5 PC |
556 | static void rec_ADDIU(struct lightrec_cstate *state, |
557 | const struct block *block, u16 offset) | |
d16005f8 | 558 | { |
684432ad PC |
559 | const struct opcode *op = &block->opcode_list[offset]; |
560 | ||
d16005f8 | 561 | _jit_name(block->_jit, __func__); |
02487de7 | 562 | |
684432ad | 563 | if (op->i.rs && !(op->flags & LIGHTREC_MOVI)) |
02487de7 PC |
564 | rec_alu_imm(state, block, offset, jit_code_addi, false); |
565 | else | |
566 | rec_movi(state, block, offset); | |
d16005f8 PC |
567 | } |
568 | ||
98fa08a5 PC |
569 | static void rec_ADDI(struct lightrec_cstate *state, |
570 | const struct block *block, u16 offset) | |
d16005f8 PC |
571 | { |
572 | /* TODO: Handle the exception? */ | |
573 | _jit_name(block->_jit, __func__); | |
02487de7 | 574 | rec_ADDIU(state, block, offset); |
d16005f8 PC |
575 | } |
576 | ||
98fa08a5 PC |
577 | static void rec_SLTIU(struct lightrec_cstate *state, |
578 | const struct block *block, u16 offset) | |
d16005f8 PC |
579 | { |
580 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 581 | rec_alu_imm(state, block, offset, jit_code_lti_u, true); |
d16005f8 PC |
582 | } |
583 | ||
98fa08a5 PC |
584 | static void rec_SLTI(struct lightrec_cstate *state, |
585 | const struct block *block, u16 offset) | |
d16005f8 PC |
586 | { |
587 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 588 | rec_alu_imm(state, block, offset, jit_code_lti, true); |
d16005f8 PC |
589 | } |
590 | ||
98fa08a5 PC |
591 | static void rec_ANDI(struct lightrec_cstate *state, |
592 | const struct block *block, u16 offset) | |
d16005f8 | 593 | { |
98fa08a5 PC |
594 | struct regcache *reg_cache = state->reg_cache; |
595 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
596 | jit_state_t *_jit = block->_jit; |
597 | u8 rs, rt; | |
598 | ||
599 | _jit_name(block->_jit, __func__); | |
600 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
601 | |
602 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
603 | c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt); | |
d16005f8 PC |
604 | |
605 | /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically | |
606 | * casts to uint8_t / uint16_t. */ | |
98fa08a5 | 607 | if (c.i.imm == 0xff) |
d16005f8 | 608 | jit_extr_uc(rt, rs); |
98fa08a5 | 609 | else if (c.i.imm == 0xffff) |
d16005f8 PC |
610 | jit_extr_us(rt, rs); |
611 | else | |
98fa08a5 PC |
612 | jit_andi(rt, rs, (u32)(u16) c.i.imm); |
613 | ||
614 | lightrec_free_reg(reg_cache, rs); | |
615 | lightrec_free_reg(reg_cache, rt); | |
616 | } | |
617 | ||
618 | static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block, | |
619 | u16 offset, jit_code_t code) | |
620 | { | |
621 | struct regcache *reg_cache = state->reg_cache; | |
622 | union code c = block->opcode_list[offset].c; | |
623 | jit_state_t *_jit = block->_jit; | |
624 | u8 rs, rt, flags; | |
625 | ||
626 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
627 | |
628 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
629 | c.i.rs, c.i.rt, 0, 0, &rs, &rt); | |
98fa08a5 PC |
630 | |
631 | flags = lightrec_get_reg_in_flags(reg_cache, rs); | |
632 | lightrec_set_reg_out_flags(reg_cache, rt, flags); | |
633 | ||
634 | jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm); | |
d16005f8 PC |
635 | |
636 | lightrec_free_reg(reg_cache, rs); | |
637 | lightrec_free_reg(reg_cache, rt); | |
638 | } | |
639 | ||
98fa08a5 PC |
640 | |
641 | static void rec_ORI(struct lightrec_cstate *state, | |
642 | const struct block *block, u16 offset) | |
d16005f8 | 643 | { |
684432ad PC |
644 | const struct opcode *op = &block->opcode_list[offset]; |
645 | struct regcache *reg_cache = state->reg_cache; | |
646 | jit_state_t *_jit = block->_jit; | |
647 | s32 val; | |
648 | u8 rt; | |
649 | ||
650 | _jit_name(_jit, __func__); | |
651 | ||
652 | if (op->flags & LIGHTREC_MOVI) { | |
653 | rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT); | |
654 | ||
655 | val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm; | |
656 | jit_movi(rt, val); | |
657 | ||
658 | lightrec_free_reg(reg_cache, rt); | |
659 | } else { | |
660 | rec_alu_or_xor(state, block, offset, jit_code_ori); | |
661 | } | |
d16005f8 PC |
662 | } |
663 | ||
98fa08a5 PC |
664 | static void rec_XORI(struct lightrec_cstate *state, |
665 | const struct block *block, u16 offset) | |
d16005f8 PC |
666 | { |
667 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 668 | rec_alu_or_xor(state, block, offset, jit_code_xori); |
d16005f8 PC |
669 | } |
670 | ||
98fa08a5 PC |
671 | static void rec_LUI(struct lightrec_cstate *state, |
672 | const struct block *block, u16 offset) | |
d16005f8 | 673 | { |
98fa08a5 PC |
674 | struct regcache *reg_cache = state->reg_cache; |
675 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 676 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 677 | u8 rt, flags = REG_EXT; |
d16005f8 | 678 | |
684432ad PC |
679 | if (block->opcode_list[offset].flags & LIGHTREC_MOVI) { |
680 | state->movi_temp[c.i.rt] = c.i.imm; | |
681 | return; | |
682 | } | |
683 | ||
d16005f8 PC |
684 | jit_name(__func__); |
685 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 686 | |
98fa08a5 PC |
687 | if (!(c.i.imm & BIT(15))) |
688 | flags |= REG_ZEXT; | |
689 | ||
690 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); | |
691 | ||
692 | jit_movi(rt, (s32)(c.i.imm << 16)); | |
d16005f8 PC |
693 | |
694 | lightrec_free_reg(reg_cache, rt); | |
695 | } | |
696 | ||
98fa08a5 PC |
697 | static void rec_special_ADDU(struct lightrec_cstate *state, |
698 | const struct block *block, u16 offset) | |
d16005f8 PC |
699 | { |
700 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 701 | rec_alu_special(state, block, offset, jit_code_addr, false); |
d16005f8 PC |
702 | } |
703 | ||
98fa08a5 PC |
704 | static void rec_special_ADD(struct lightrec_cstate *state, |
705 | const struct block *block, u16 offset) | |
d16005f8 PC |
706 | { |
707 | /* TODO: Handle the exception? */ | |
708 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 709 | rec_alu_special(state, block, offset, jit_code_addr, false); |
d16005f8 PC |
710 | } |
711 | ||
98fa08a5 PC |
712 | static void rec_special_SUBU(struct lightrec_cstate *state, |
713 | const struct block *block, u16 offset) | |
d16005f8 PC |
714 | { |
715 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 716 | rec_alu_special(state, block, offset, jit_code_subr, false); |
d16005f8 PC |
717 | } |
718 | ||
98fa08a5 PC |
719 | static void rec_special_SUB(struct lightrec_cstate *state, |
720 | const struct block *block, u16 offset) | |
d16005f8 PC |
721 | { |
722 | /* TODO: Handle the exception? */ | |
723 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 724 | rec_alu_special(state, block, offset, jit_code_subr, false); |
d16005f8 PC |
725 | } |
726 | ||
98fa08a5 PC |
727 | static void rec_special_AND(struct lightrec_cstate *state, |
728 | const struct block *block, u16 offset) | |
d16005f8 | 729 | { |
98fa08a5 PC |
730 | struct regcache *reg_cache = state->reg_cache; |
731 | union code c = block->opcode_list[offset].c; | |
732 | jit_state_t *_jit = block->_jit; | |
733 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; | |
734 | ||
d16005f8 | 735 | _jit_name(block->_jit, __func__); |
98fa08a5 | 736 | jit_note(__FILE__, __LINE__); |
684432ad | 737 | |
98fa08a5 | 738 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
739 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
740 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
98fa08a5 PC |
741 | |
742 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); | |
743 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
744 | ||
745 | /* Z(rd) = Z(rs) | Z(rt) */ | |
746 | flags_rd = REG_ZEXT & (flags_rs | flags_rt); | |
747 | ||
748 | /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */ | |
749 | if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) || | |
750 | ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) || | |
751 | (REG_EXT & flags_rs & flags_rt)) | |
752 | flags_rd |= REG_EXT; | |
753 | ||
754 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
755 | ||
756 | jit_andr(rd, rs, rt); | |
757 | ||
758 | lightrec_free_reg(reg_cache, rs); | |
759 | lightrec_free_reg(reg_cache, rt); | |
760 | lightrec_free_reg(reg_cache, rd); | |
d16005f8 PC |
761 | } |
762 | ||
98fa08a5 PC |
763 | static void rec_special_or_nor(struct lightrec_cstate *state, |
764 | const struct block *block, u16 offset, bool nor) | |
765 | { | |
766 | struct regcache *reg_cache = state->reg_cache; | |
767 | union code c = block->opcode_list[offset].c; | |
768 | jit_state_t *_jit = block->_jit; | |
769 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0; | |
770 | ||
771 | jit_note(__FILE__, __LINE__); | |
684432ad | 772 | |
98fa08a5 | 773 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
774 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
775 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
98fa08a5 PC |
776 | |
777 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); | |
778 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
779 | ||
780 | /* or: Z(rd) = Z(rs) & Z(rt) | |
781 | * nor: Z(rd) = 0 */ | |
782 | if (!nor) | |
783 | flags_rd = REG_ZEXT & flags_rs & flags_rt; | |
784 | ||
6ce0b00a PC |
785 | /* E(rd) = E(rs) & E(rt) */ |
786 | if (REG_EXT & flags_rs & flags_rt) | |
98fa08a5 PC |
787 | flags_rd |= REG_EXT; |
788 | ||
789 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
790 | ||
791 | jit_orr(rd, rs, rt); | |
792 | ||
793 | if (nor) | |
794 | jit_comr(rd, rd); | |
795 | ||
796 | lightrec_free_reg(reg_cache, rs); | |
797 | lightrec_free_reg(reg_cache, rt); | |
798 | lightrec_free_reg(reg_cache, rd); | |
799 | } | |
800 | ||
801 | static void rec_special_OR(struct lightrec_cstate *state, | |
802 | const struct block *block, u16 offset) | |
d16005f8 PC |
803 | { |
804 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 805 | rec_special_or_nor(state, block, offset, false); |
d16005f8 PC |
806 | } |
807 | ||
98fa08a5 PC |
808 | static void rec_special_NOR(struct lightrec_cstate *state, |
809 | const struct block *block, u16 offset) | |
d16005f8 PC |
810 | { |
811 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 812 | rec_special_or_nor(state, block, offset, true); |
d16005f8 PC |
813 | } |
814 | ||
98fa08a5 PC |
815 | static void rec_special_XOR(struct lightrec_cstate *state, |
816 | const struct block *block, u16 offset) | |
d16005f8 | 817 | { |
98fa08a5 PC |
818 | struct regcache *reg_cache = state->reg_cache; |
819 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 820 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 821 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; |
d16005f8 | 822 | |
98fa08a5 PC |
823 | _jit_name(block->_jit, __func__); |
824 | ||
825 | jit_note(__FILE__, __LINE__); | |
684432ad | 826 | |
98fa08a5 | 827 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
828 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
829 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
d16005f8 | 830 | |
98fa08a5 PC |
831 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); |
832 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
d16005f8 | 833 | |
98fa08a5 PC |
834 | /* Z(rd) = Z(rs) & Z(rt) */ |
835 | flags_rd = REG_ZEXT & flags_rs & flags_rt; | |
836 | ||
837 | /* E(rd) = E(rs) & E(rt) */ | |
838 | flags_rd |= REG_EXT & flags_rs & flags_rt; | |
839 | ||
840 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
841 | ||
842 | jit_xorr(rd, rs, rt); | |
843 | ||
844 | lightrec_free_reg(reg_cache, rs); | |
845 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
846 | lightrec_free_reg(reg_cache, rd); |
847 | } | |
848 | ||
98fa08a5 PC |
849 | static void rec_special_SLTU(struct lightrec_cstate *state, |
850 | const struct block *block, u16 offset) | |
d16005f8 PC |
851 | { |
852 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 853 | rec_alu_special(state, block, offset, jit_code_ltr_u, true); |
d16005f8 PC |
854 | } |
855 | ||
98fa08a5 PC |
856 | static void rec_special_SLT(struct lightrec_cstate *state, |
857 | const struct block *block, u16 offset) | |
d16005f8 PC |
858 | { |
859 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 860 | rec_alu_special(state, block, offset, jit_code_ltr, true); |
d16005f8 PC |
861 | } |
862 | ||
98fa08a5 PC |
863 | static void rec_special_SLLV(struct lightrec_cstate *state, |
864 | const struct block *block, u16 offset) | |
d16005f8 PC |
865 | { |
866 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 867 | rec_alu_shiftv(state, block, offset, jit_code_lshr); |
d16005f8 PC |
868 | } |
869 | ||
98fa08a5 PC |
870 | static void rec_special_SRLV(struct lightrec_cstate *state, |
871 | const struct block *block, u16 offset) | |
d16005f8 PC |
872 | { |
873 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 874 | rec_alu_shiftv(state, block, offset, jit_code_rshr_u); |
d16005f8 PC |
875 | } |
876 | ||
98fa08a5 PC |
877 | static void rec_special_SRAV(struct lightrec_cstate *state, |
878 | const struct block *block, u16 offset) | |
d16005f8 PC |
879 | { |
880 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 881 | rec_alu_shiftv(state, block, offset, jit_code_rshr); |
d16005f8 PC |
882 | } |
883 | ||
98fa08a5 PC |
884 | static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block, |
885 | u16 offset, jit_code_t code) | |
d16005f8 | 886 | { |
98fa08a5 PC |
887 | struct regcache *reg_cache = state->reg_cache; |
888 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 889 | jit_state_t *_jit = block->_jit; |
684432ad | 890 | u8 rd, rt, flags = 0, out_flags = 0; |
d16005f8 PC |
891 | |
892 | jit_note(__FILE__, __LINE__); | |
893 | ||
98fa08a5 PC |
894 | if (code == jit_code_rshi) |
895 | flags = REG_EXT; | |
896 | else if (code == jit_code_rshi_u) | |
897 | flags = REG_ZEXT; | |
d16005f8 | 898 | |
98fa08a5 PC |
899 | /* Input reg is zero-extended, if we SRL at least by one bit, we know |
900 | * the output reg will be both zero-extended and sign-extended. */ | |
684432ad | 901 | out_flags = flags; |
98fa08a5 | 902 | if (code == jit_code_rshi_u && c.r.imm) |
684432ad PC |
903 | out_flags |= REG_EXT; |
904 | ||
905 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
906 | c.r.rt, c.r.rd, flags, out_flags, &rt, &rd); | |
98fa08a5 PC |
907 | |
908 | jit_new_node_www(code, rd, rt, c.r.imm); | |
d16005f8 PC |
909 | |
910 | lightrec_free_reg(reg_cache, rt); | |
911 | lightrec_free_reg(reg_cache, rd); | |
912 | } | |
913 | ||
98fa08a5 PC |
914 | static void rec_special_SLL(struct lightrec_cstate *state, |
915 | const struct block *block, u16 offset) | |
d16005f8 PC |
916 | { |
917 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 918 | rec_alu_shift(state, block, offset, jit_code_lshi); |
d16005f8 PC |
919 | } |
920 | ||
98fa08a5 PC |
921 | static void rec_special_SRL(struct lightrec_cstate *state, |
922 | const struct block *block, u16 offset) | |
d16005f8 PC |
923 | { |
924 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 925 | rec_alu_shift(state, block, offset, jit_code_rshi_u); |
d16005f8 PC |
926 | } |
927 | ||
98fa08a5 PC |
928 | static void rec_special_SRA(struct lightrec_cstate *state, |
929 | const struct block *block, u16 offset) | |
d16005f8 PC |
930 | { |
931 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 932 | rec_alu_shift(state, block, offset, jit_code_rshi); |
d16005f8 PC |
933 | } |
934 | ||
98fa08a5 PC |
935 | static void rec_alu_mult(struct lightrec_cstate *state, |
936 | const struct block *block, u16 offset, bool is_signed) | |
d16005f8 | 937 | { |
98fa08a5 PC |
938 | struct regcache *reg_cache = state->reg_cache; |
939 | union code c = block->opcode_list[offset].c; | |
03535202 | 940 | u32 flags = block->opcode_list[offset].flags; |
98fa08a5 PC |
941 | u8 reg_lo = get_mult_div_lo(c); |
942 | u8 reg_hi = get_mult_div_hi(c); | |
d16005f8 | 943 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 944 | u8 lo, hi, rs, rt, rflags = 0; |
d16005f8 PC |
945 | |
946 | jit_note(__FILE__, __LINE__); | |
947 | ||
98fa08a5 PC |
948 | if (is_signed) |
949 | rflags = REG_EXT; | |
950 | else | |
951 | rflags = REG_ZEXT; | |
952 | ||
953 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); | |
954 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); | |
955 | ||
03535202 | 956 | if (!op_flag_no_lo(flags)) |
98fa08a5 PC |
957 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); |
958 | else if (__WORDSIZE == 32) | |
959 | lo = lightrec_alloc_reg_temp(reg_cache, _jit); | |
960 | ||
03535202 | 961 | if (!op_flag_no_hi(flags)) |
98fa08a5 PC |
962 | hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT); |
963 | ||
964 | if (__WORDSIZE == 32) { | |
965 | /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit | |
966 | * operation if the MULT was detected a 32-bit only. */ | |
03535202 | 967 | if (!op_flag_no_hi(flags)) { |
98fa08a5 PC |
968 | if (is_signed) |
969 | jit_qmulr(lo, hi, rs, rt); | |
970 | else | |
971 | jit_qmulr_u(lo, hi, rs, rt); | |
972 | } else { | |
973 | jit_mulr(lo, rs, rt); | |
974 | } | |
d16005f8 | 975 | } else { |
98fa08a5 | 976 | /* On 64-bit systems, do a 64*64->64 bit operation. */ |
03535202 | 977 | if (op_flag_no_lo(flags)) { |
98fa08a5 PC |
978 | jit_mulr(hi, rs, rt); |
979 | jit_rshi(hi, hi, 32); | |
980 | } else { | |
981 | jit_mulr(lo, rs, rt); | |
d16005f8 | 982 | |
98fa08a5 | 983 | /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */ |
03535202 | 984 | if (!op_flag_no_hi(flags)) |
98fa08a5 PC |
985 | jit_rshi(hi, lo, 32); |
986 | } | |
d16005f8 PC |
987 | } |
988 | ||
d16005f8 PC |
989 | lightrec_free_reg(reg_cache, rs); |
990 | lightrec_free_reg(reg_cache, rt); | |
03535202 | 991 | if (!op_flag_no_lo(flags) || __WORDSIZE == 32) |
98fa08a5 | 992 | lightrec_free_reg(reg_cache, lo); |
03535202 | 993 | if (!op_flag_no_hi(flags)) |
d16005f8 PC |
994 | lightrec_free_reg(reg_cache, hi); |
995 | } | |
996 | ||
98fa08a5 PC |
997 | static void rec_alu_div(struct lightrec_cstate *state, |
998 | const struct block *block, u16 offset, bool is_signed) | |
d16005f8 | 999 | { |
98fa08a5 PC |
1000 | struct regcache *reg_cache = state->reg_cache; |
1001 | union code c = block->opcode_list[offset].c; | |
03535202 PC |
1002 | u32 flags = block->opcode_list[offset].flags; |
1003 | bool no_check = op_flag_no_div_check(flags); | |
98fa08a5 PC |
1004 | u8 reg_lo = get_mult_div_lo(c); |
1005 | u8 reg_hi = get_mult_div_hi(c); | |
d16005f8 PC |
1006 | jit_state_t *_jit = block->_jit; |
1007 | jit_node_t *branch, *to_end; | |
fd58fa32 | 1008 | u8 lo = 0, hi = 0, rs, rt, rflags = 0; |
d16005f8 PC |
1009 | |
1010 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 1011 | |
98fa08a5 PC |
1012 | if (is_signed) |
1013 | rflags = REG_EXT; | |
1014 | else | |
1015 | rflags = REG_ZEXT; | |
1016 | ||
1017 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); | |
1018 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); | |
1019 | ||
03535202 | 1020 | if (!op_flag_no_lo(flags)) |
98fa08a5 PC |
1021 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); |
1022 | ||
03535202 | 1023 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1024 | hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0); |
d16005f8 PC |
1025 | |
1026 | /* Jump to special handler if dividing by zero */ | |
98fa08a5 PC |
1027 | if (!no_check) |
1028 | branch = jit_beqi(rt, 0); | |
d16005f8 | 1029 | |
03535202 | 1030 | if (op_flag_no_lo(flags)) { |
98fa08a5 PC |
1031 | if (is_signed) |
1032 | jit_remr(hi, rs, rt); | |
1033 | else | |
1034 | jit_remr_u(hi, rs, rt); | |
03535202 | 1035 | } else if (op_flag_no_hi(flags)) { |
98fa08a5 PC |
1036 | if (is_signed) |
1037 | jit_divr(lo, rs, rt); | |
1038 | else | |
1039 | jit_divr_u(lo, rs, rt); | |
d16005f8 | 1040 | } else { |
98fa08a5 PC |
1041 | if (is_signed) |
1042 | jit_qdivr(lo, hi, rs, rt); | |
1043 | else | |
1044 | jit_qdivr_u(lo, hi, rs, rt); | |
d16005f8 | 1045 | } |
d16005f8 | 1046 | |
98fa08a5 | 1047 | if (!no_check) { |
98fa08a5 | 1048 | /* Jump above the div-by-zero handler */ |
22eee2ac | 1049 | to_end = jit_b(); |
d16005f8 | 1050 | |
98fa08a5 PC |
1051 | jit_patch(branch); |
1052 | ||
03535202 | 1053 | if (!op_flag_no_lo(flags)) { |
98fa08a5 | 1054 | if (is_signed) { |
9259d748 | 1055 | jit_ltr(lo, rs, rt); |
98fa08a5 PC |
1056 | jit_lshi(lo, lo, 1); |
1057 | jit_subi(lo, lo, 1); | |
1058 | } else { | |
9259d748 | 1059 | jit_subi(lo, rt, 1); |
98fa08a5 PC |
1060 | } |
1061 | } | |
d16005f8 | 1062 | |
03535202 | 1063 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1064 | jit_movr(hi, rs); |
d16005f8 | 1065 | |
98fa08a5 PC |
1066 | jit_patch(to_end); |
1067 | } | |
d16005f8 PC |
1068 | |
1069 | lightrec_free_reg(reg_cache, rs); | |
1070 | lightrec_free_reg(reg_cache, rt); | |
98fa08a5 | 1071 | |
03535202 | 1072 | if (!op_flag_no_lo(flags)) |
98fa08a5 PC |
1073 | lightrec_free_reg(reg_cache, lo); |
1074 | ||
03535202 | 1075 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1076 | lightrec_free_reg(reg_cache, hi); |
d16005f8 PC |
1077 | } |
1078 | ||
98fa08a5 PC |
1079 | static void rec_special_MULT(struct lightrec_cstate *state, |
1080 | const struct block *block, u16 offset) | |
d16005f8 PC |
1081 | { |
1082 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1083 | rec_alu_mult(state, block, offset, true); |
d16005f8 PC |
1084 | } |
1085 | ||
98fa08a5 PC |
1086 | static void rec_special_MULTU(struct lightrec_cstate *state, |
1087 | const struct block *block, u16 offset) | |
d16005f8 PC |
1088 | { |
1089 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1090 | rec_alu_mult(state, block, offset, false); |
d16005f8 PC |
1091 | } |
1092 | ||
98fa08a5 PC |
1093 | static void rec_special_DIV(struct lightrec_cstate *state, |
1094 | const struct block *block, u16 offset) | |
d16005f8 PC |
1095 | { |
1096 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1097 | rec_alu_div(state, block, offset, true); |
d16005f8 PC |
1098 | } |
1099 | ||
98fa08a5 PC |
1100 | static void rec_special_DIVU(struct lightrec_cstate *state, |
1101 | const struct block *block, u16 offset) | |
d16005f8 PC |
1102 | { |
1103 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1104 | rec_alu_div(state, block, offset, false); |
d16005f8 PC |
1105 | } |
1106 | ||
98fa08a5 | 1107 | static void rec_alu_mv_lo_hi(struct lightrec_cstate *state, |
684432ad PC |
1108 | const struct block *block, u16 offset, |
1109 | u8 dst, u8 src) | |
d16005f8 | 1110 | { |
98fa08a5 | 1111 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 PC |
1112 | jit_state_t *_jit = block->_jit; |
1113 | ||
1114 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
1115 | |
1116 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
1117 | src, dst, 0, REG_EXT, &src, &dst); | |
d16005f8 | 1118 | |
d16005f8 | 1119 | jit_extr_i(dst, src); |
d16005f8 PC |
1120 | |
1121 | lightrec_free_reg(reg_cache, src); | |
1122 | lightrec_free_reg(reg_cache, dst); | |
1123 | } | |
1124 | ||
98fa08a5 PC |
1125 | static void rec_special_MFHI(struct lightrec_cstate *state, |
1126 | const struct block *block, u16 offset) | |
d16005f8 | 1127 | { |
98fa08a5 PC |
1128 | union code c = block->opcode_list[offset].c; |
1129 | ||
d16005f8 | 1130 | _jit_name(block->_jit, __func__); |
684432ad | 1131 | rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI); |
d16005f8 PC |
1132 | } |
1133 | ||
98fa08a5 PC |
1134 | static void rec_special_MTHI(struct lightrec_cstate *state, |
1135 | const struct block *block, u16 offset) | |
d16005f8 | 1136 | { |
98fa08a5 PC |
1137 | union code c = block->opcode_list[offset].c; |
1138 | ||
d16005f8 | 1139 | _jit_name(block->_jit, __func__); |
684432ad | 1140 | rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs); |
d16005f8 PC |
1141 | } |
1142 | ||
98fa08a5 PC |
1143 | static void rec_special_MFLO(struct lightrec_cstate *state, |
1144 | const struct block *block, u16 offset) | |
d16005f8 | 1145 | { |
98fa08a5 PC |
1146 | union code c = block->opcode_list[offset].c; |
1147 | ||
d16005f8 | 1148 | _jit_name(block->_jit, __func__); |
684432ad | 1149 | rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO); |
d16005f8 PC |
1150 | } |
1151 | ||
98fa08a5 PC |
1152 | static void rec_special_MTLO(struct lightrec_cstate *state, |
1153 | const struct block *block, u16 offset) | |
d16005f8 | 1154 | { |
98fa08a5 PC |
1155 | union code c = block->opcode_list[offset].c; |
1156 | ||
d16005f8 | 1157 | _jit_name(block->_jit, __func__); |
684432ad | 1158 | rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs); |
d16005f8 PC |
1159 | } |
1160 | ||
ba3814c1 PC |
1161 | static void call_to_c_wrapper(struct lightrec_cstate *state, |
1162 | const struct block *block, u32 arg, | |
1163 | enum c_wrappers wrapper) | |
d16005f8 | 1164 | { |
98fa08a5 | 1165 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 1166 | jit_state_t *_jit = block->_jit; |
ba3814c1 | 1167 | s8 tmp, tmp2; |
d16005f8 | 1168 | |
ba3814c1 PC |
1169 | /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */ |
1170 | tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1); | |
d16005f8 | 1171 | |
ba3814c1 PC |
1172 | tmp = lightrec_get_reg_with_value(reg_cache, |
1173 | (intptr_t) state->state->wrappers_eps[wrapper]); | |
1174 | if (tmp < 0) { | |
1175 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
1176 | jit_ldxi(tmp, LIGHTREC_REG_STATE, | |
1177 | offsetof(struct lightrec_state, wrappers_eps[wrapper])); | |
1178 | ||
1179 | lightrec_temp_set_value(reg_cache, tmp, | |
1180 | (intptr_t) state->state->wrappers_eps[wrapper]); | |
22eee2ac PC |
1181 | } |
1182 | ||
ba3814c1 PC |
1183 | lightrec_free_reg(reg_cache, tmp2); |
1184 | ||
1185 | #ifdef __mips__ | |
1186 | /* On MIPS, register t9 is always used as the target register for JALR. | |
1187 | * Therefore if it does not contain the target address we must | |
1188 | * invalidate it. */ | |
1189 | if (tmp != _T9) | |
1190 | lightrec_unload_reg(reg_cache, _jit, _T9); | |
1191 | #endif | |
1192 | ||
1193 | jit_prepare(); | |
1194 | jit_pushargi(arg); | |
1195 | ||
22eee2ac | 1196 | lightrec_regcache_mark_live(reg_cache, _jit); |
98fa08a5 | 1197 | jit_callr(tmp); |
d16005f8 | 1198 | |
98fa08a5 | 1199 | lightrec_free_reg(reg_cache, tmp); |
98fa08a5 PC |
1200 | lightrec_regcache_mark_live(reg_cache, _jit); |
1201 | } | |
1202 | ||
1203 | static void rec_io(struct lightrec_cstate *state, | |
1204 | const struct block *block, u16 offset, | |
1205 | bool load_rt, bool read_rt) | |
1206 | { | |
1207 | struct regcache *reg_cache = state->reg_cache; | |
1208 | jit_state_t *_jit = block->_jit; | |
1209 | union code c = block->opcode_list[offset].c; | |
03535202 | 1210 | u32 flags = block->opcode_list[offset].flags; |
22eee2ac | 1211 | bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags); |
98fa08a5 | 1212 | u32 lut_entry; |
cb72ea13 | 1213 | u8 zero; |
d16005f8 | 1214 | |
98fa08a5 | 1215 | jit_note(__FILE__, __LINE__); |
d16005f8 | 1216 | |
98fa08a5 PC |
1217 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); |
1218 | ||
1219 | if (read_rt && likely(c.i.rt)) | |
1220 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); | |
d16005f8 | 1221 | else if (load_rt) |
98fa08a5 | 1222 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); |
d16005f8 | 1223 | |
cb72ea13 PC |
1224 | if (op_flag_load_delay(flags) && !state->no_load_delay) { |
1225 | /* Clear state->in_delay_slot_n. This notifies the lightrec_rw | |
1226 | * wrapper that it should write the REG_TEMP register instead of | |
1227 | * the actual output register of the opcode. */ | |
1228 | zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); | |
1229 | jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n), | |
1230 | LIGHTREC_REG_STATE, zero); | |
1231 | lightrec_free_reg(reg_cache, zero); | |
1232 | } | |
1233 | ||
d16005f8 | 1234 | if (is_tagged) { |
ba3814c1 | 1235 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW); |
d16005f8 | 1236 | } else { |
98fa08a5 PC |
1237 | lut_entry = lightrec_get_lut_entry(block); |
1238 | call_to_c_wrapper(state, block, (lut_entry << 16) | offset, | |
ba3814c1 | 1239 | C_WRAPPER_RW_GENERIC); |
d16005f8 | 1240 | } |
d16005f8 PC |
1241 | } |
1242 | ||
02487de7 PC |
1243 | static u32 rec_ram_mask(struct lightrec_state *state) |
1244 | { | |
1245 | return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1; | |
1246 | } | |
1247 | ||
13b02197 PC |
1248 | static u32 rec_io_mask(const struct lightrec_state *state) |
1249 | { | |
1250 | u32 length = state->maps[PSX_MAP_HW_REGISTERS].length; | |
1251 | ||
2ec79b77 | 1252 | return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0); |
13b02197 PC |
1253 | } |
1254 | ||
22eee2ac PC |
1255 | static void rec_store_memory(struct lightrec_cstate *cstate, |
1256 | const struct block *block, | |
1257 | u16 offset, jit_code_t code, | |
02487de7 | 1258 | jit_code_t swap_code, |
22eee2ac PC |
1259 | uintptr_t addr_offset, u32 addr_mask, |
1260 | bool invalidate) | |
1261 | { | |
02487de7 | 1262 | const struct lightrec_state *state = cstate->state; |
22eee2ac PC |
1263 | struct regcache *reg_cache = cstate->reg_cache; |
1264 | struct opcode *op = &block->opcode_list[offset]; | |
1265 | jit_state_t *_jit = block->_jit; | |
1266 | union code c = op->c; | |
1267 | u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2; | |
1268 | s16 imm = (s16)c.i.imm; | |
02487de7 | 1269 | s32 simm = (s32)imm << (1 - lut_is_32bit(state)); |
22eee2ac | 1270 | s32 lut_offt = offsetof(struct lightrec_state, code_lut); |
03535202 | 1271 | bool no_mask = op_flag_no_mask(op->flags); |
02487de7 PC |
1272 | bool add_imm = c.i.imm && |
1273 | ((!state->mirrors_mapped && !no_mask) || (invalidate && | |
1274 | ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt)))); | |
684432ad | 1275 | bool need_tmp = !no_mask || add_imm || invalidate; |
9259d748 | 1276 | bool swc2 = c.i.op == OP_SWC2; |
cb72ea13 | 1277 | u8 in_reg = swc2 ? REG_TEMP : c.i.rt; |
684432ad | 1278 | s8 reg_imm; |
22eee2ac | 1279 | |
22eee2ac PC |
1280 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
1281 | if (need_tmp) | |
1282 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
1283 | ||
1284 | addr_reg = rs; | |
1285 | ||
1286 | if (add_imm) { | |
1287 | jit_addi(tmp, addr_reg, (s16)c.i.imm); | |
0e720fb1 | 1288 | lightrec_free_reg(reg_cache, rs); |
22eee2ac PC |
1289 | addr_reg = tmp; |
1290 | imm = 0; | |
1291 | } else if (simm) { | |
1292 | lut_offt += simm; | |
1293 | } | |
1294 | ||
1295 | if (!no_mask) { | |
684432ad PC |
1296 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1297 | addr_mask); | |
1298 | ||
1299 | jit_andr(tmp, addr_reg, reg_imm); | |
22eee2ac | 1300 | addr_reg = tmp; |
684432ad PC |
1301 | |
1302 | lightrec_free_reg(reg_cache, reg_imm); | |
22eee2ac PC |
1303 | } |
1304 | ||
22eee2ac | 1305 | if (addr_offset) { |
684432ad PC |
1306 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1307 | addr_offset); | |
ba3814c1 | 1308 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
684432ad | 1309 | jit_addr(tmp2, addr_reg, reg_imm); |
22eee2ac | 1310 | addr_reg2 = tmp2; |
684432ad PC |
1311 | |
1312 | lightrec_free_reg(reg_cache, reg_imm); | |
22eee2ac PC |
1313 | } else { |
1314 | addr_reg2 = addr_reg; | |
1315 | } | |
1316 | ||
684432ad PC |
1317 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
1318 | ||
9259d748 | 1319 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1320 | tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1321 | ||
1322 | jit_new_node_ww(swap_code, tmp3, rt); | |
1323 | jit_new_node_www(code, imm, addr_reg2, tmp3); | |
1324 | ||
1325 | lightrec_free_reg(reg_cache, tmp3); | |
1326 | } else { | |
1327 | jit_new_node_www(code, imm, addr_reg2, rt); | |
1328 | } | |
1329 | ||
22eee2ac PC |
1330 | lightrec_free_reg(reg_cache, rt); |
1331 | ||
1332 | if (invalidate) { | |
1333 | tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); | |
1334 | ||
1335 | if (c.i.op != OP_SW) { | |
ba3814c1 PC |
1336 | jit_andi(tmp, addr_reg, ~3); |
1337 | addr_reg = tmp; | |
22eee2ac PC |
1338 | } |
1339 | ||
02487de7 | 1340 | if (!lut_is_32bit(state)) { |
ba3814c1 PC |
1341 | jit_lshi(tmp, addr_reg, 1); |
1342 | addr_reg = tmp; | |
22eee2ac PC |
1343 | } |
1344 | ||
02487de7 PC |
1345 | if (addr_reg == rs && c.i.rs == 0) { |
1346 | addr_reg = LIGHTREC_REG_STATE; | |
1347 | } else { | |
cb72ea13 | 1348 | jit_add_state(tmp, addr_reg); |
ba3814c1 | 1349 | addr_reg = tmp; |
22eee2ac PC |
1350 | } |
1351 | ||
02487de7 PC |
1352 | if (lut_is_32bit(state)) |
1353 | jit_stxi_i(lut_offt, addr_reg, tmp3); | |
1354 | else | |
1355 | jit_stxi(lut_offt, addr_reg, tmp3); | |
22eee2ac PC |
1356 | |
1357 | lightrec_free_reg(reg_cache, tmp3); | |
1358 | } | |
1359 | ||
ba3814c1 | 1360 | if (addr_offset) |
22eee2ac PC |
1361 | lightrec_free_reg(reg_cache, tmp2); |
1362 | if (need_tmp) | |
1363 | lightrec_free_reg(reg_cache, tmp); | |
1364 | lightrec_free_reg(reg_cache, rs); | |
1365 | } | |
1366 | ||
1367 | static void rec_store_ram(struct lightrec_cstate *cstate, | |
1368 | const struct block *block, | |
1369 | u16 offset, jit_code_t code, | |
02487de7 | 1370 | jit_code_t swap_code, bool invalidate) |
22eee2ac | 1371 | { |
02487de7 PC |
1372 | struct lightrec_state *state = cstate->state; |
1373 | ||
22eee2ac PC |
1374 | _jit_note(block->_jit, __FILE__, __LINE__); |
1375 | ||
02487de7 PC |
1376 | return rec_store_memory(cstate, block, offset, code, swap_code, |
1377 | state->offset_ram, rec_ram_mask(state), | |
1378 | invalidate); | |
22eee2ac PC |
1379 | } |
1380 | ||
1381 | static void rec_store_scratch(struct lightrec_cstate *cstate, | |
02487de7 PC |
1382 | const struct block *block, u16 offset, |
1383 | jit_code_t code, jit_code_t swap_code) | |
22eee2ac PC |
1384 | { |
1385 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1386 | ||
02487de7 | 1387 | return rec_store_memory(cstate, block, offset, code, swap_code, |
22eee2ac PC |
1388 | cstate->state->offset_scratch, |
1389 | 0x1fffffff, false); | |
1390 | } | |
1391 | ||
ba3814c1 PC |
1392 | static void rec_store_io(struct lightrec_cstate *cstate, |
1393 | const struct block *block, u16 offset, | |
1394 | jit_code_t code, jit_code_t swap_code) | |
1395 | { | |
1396 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1397 | ||
1398 | return rec_store_memory(cstate, block, offset, code, swap_code, | |
1399 | cstate->state->offset_io, | |
13b02197 | 1400 | rec_io_mask(cstate->state), false); |
ba3814c1 PC |
1401 | } |
1402 | ||
98fa08a5 PC |
1403 | static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate, |
1404 | const struct block *block, | |
02487de7 PC |
1405 | u16 offset, jit_code_t code, |
1406 | jit_code_t swap_code) | |
d16005f8 | 1407 | { |
98fa08a5 PC |
1408 | struct lightrec_state *state = cstate->state; |
1409 | struct regcache *reg_cache = cstate->reg_cache; | |
1410 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
1411 | jit_state_t *_jit = block->_jit; |
1412 | jit_node_t *to_not_ram, *to_end; | |
9259d748 | 1413 | bool swc2 = c.i.op == OP_SWC2; |
684432ad PC |
1414 | u8 tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt; |
1415 | u32 addr_mask; | |
1416 | s32 reg_imm; | |
d16005f8 PC |
1417 | s16 imm; |
1418 | ||
1419 | jit_note(__FILE__, __LINE__); | |
98fa08a5 | 1420 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
d16005f8 | 1421 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
98fa08a5 | 1422 | |
684432ad PC |
1423 | if (state->mirrors_mapped) |
1424 | addr_mask = 0x1f800000 | (4 * RAM_SIZE - 1); | |
1425 | else | |
1426 | addr_mask = 0x1f800000 | (RAM_SIZE - 1); | |
1427 | ||
1428 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, addr_mask); | |
d16005f8 PC |
1429 | |
1430 | /* Convert to KUNSEG and avoid RAM mirrors */ | |
684432ad | 1431 | if (!state->mirrors_mapped && c.i.imm) { |
d16005f8 | 1432 | imm = 0; |
98fa08a5 | 1433 | jit_addi(tmp, rs, (s16)c.i.imm); |
684432ad | 1434 | jit_andr(tmp, tmp, reg_imm); |
d16005f8 | 1435 | } else { |
684432ad PC |
1436 | imm = (s16)c.i.imm; |
1437 | jit_andr(tmp, rs, reg_imm); | |
d16005f8 PC |
1438 | } |
1439 | ||
1440 | lightrec_free_reg(reg_cache, rs); | |
684432ad | 1441 | lightrec_free_reg(reg_cache, reg_imm); |
d16005f8 PC |
1442 | |
1443 | if (state->offset_ram != state->offset_scratch) { | |
684432ad PC |
1444 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1445 | ||
d16005f8 PC |
1446 | to_not_ram = jit_bmsi(tmp, BIT(28)); |
1447 | ||
1448 | jit_movi(tmp2, state->offset_ram); | |
1449 | ||
22eee2ac | 1450 | to_end = jit_b(); |
d16005f8 PC |
1451 | jit_patch(to_not_ram); |
1452 | ||
1453 | jit_movi(tmp2, state->offset_scratch); | |
1454 | jit_patch(to_end); | |
1455 | } else if (state->offset_ram) { | |
684432ad PC |
1456 | tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1457 | state->offset_ram); | |
d16005f8 PC |
1458 | } |
1459 | ||
684432ad | 1460 | if (state->offset_ram || state->offset_scratch) { |
d16005f8 | 1461 | jit_addr(tmp, tmp, tmp2); |
98fa08a5 PC |
1462 | lightrec_free_reg(reg_cache, tmp2); |
1463 | } | |
d16005f8 | 1464 | |
9259d748 | 1465 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
ba3814c1 | 1466 | |
9259d748 | 1467 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1468 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1469 | ||
1470 | jit_new_node_ww(swap_code, tmp2, rt); | |
1471 | jit_new_node_www(code, imm, tmp, tmp2); | |
1472 | ||
1473 | lightrec_free_reg(reg_cache, tmp2); | |
1474 | } else { | |
1475 | jit_new_node_www(code, imm, tmp, rt); | |
1476 | } | |
d16005f8 PC |
1477 | |
1478 | lightrec_free_reg(reg_cache, rt); | |
1479 | lightrec_free_reg(reg_cache, tmp); | |
1480 | } | |
1481 | ||
98fa08a5 | 1482 | static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block, |
02487de7 | 1483 | u16 offset, jit_code_t code, jit_code_t swap_code) |
d16005f8 | 1484 | { |
98fa08a5 PC |
1485 | struct lightrec_state *state = cstate->state; |
1486 | u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE; | |
1487 | struct regcache *reg_cache = cstate->reg_cache; | |
1488 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 1489 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 1490 | jit_node_t *to_not_ram, *to_end; |
9259d748 PC |
1491 | bool swc2 = c.i.op == OP_SWC2; |
1492 | u8 tmp, tmp2, tmp3, masked_reg, rs, rt; | |
cb72ea13 | 1493 | u8 in_reg = swc2 ? REG_TEMP : c.i.rt; |
684432ad PC |
1494 | u32 addr_mask = 0x1f800000 | (ram_size - 1); |
1495 | s32 reg_imm; | |
d16005f8 PC |
1496 | |
1497 | jit_note(__FILE__, __LINE__); | |
1498 | ||
98fa08a5 | 1499 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
d16005f8 | 1500 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
98fa08a5 | 1501 | tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); |
d16005f8 | 1502 | |
684432ad PC |
1503 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, addr_mask); |
1504 | ||
d16005f8 | 1505 | /* Convert to KUNSEG and avoid RAM mirrors */ |
98fa08a5 PC |
1506 | if (c.i.imm) { |
1507 | jit_addi(tmp2, rs, (s16)c.i.imm); | |
684432ad | 1508 | jit_andr(tmp2, tmp2, reg_imm); |
d16005f8 | 1509 | } else { |
684432ad | 1510 | jit_andr(tmp2, rs, reg_imm); |
d16005f8 PC |
1511 | } |
1512 | ||
1513 | lightrec_free_reg(reg_cache, rs); | |
684432ad | 1514 | lightrec_free_reg(reg_cache, reg_imm); |
d16005f8 PC |
1515 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
1516 | ||
9259d748 PC |
1517 | if (state->offset_ram != state->offset_scratch) { |
1518 | to_not_ram = jit_bgti(tmp2, ram_size); | |
1519 | masked_reg = tmp2; | |
1520 | } else { | |
1521 | jit_lti_u(tmp, tmp2, ram_size); | |
1522 | jit_movnr(tmp, tmp2, tmp); | |
1523 | masked_reg = tmp; | |
1524 | } | |
98fa08a5 | 1525 | |
d16005f8 | 1526 | /* Compute the offset to the code LUT */ |
9259d748 PC |
1527 | if (c.i.op == OP_SW) |
1528 | jit_andi(tmp, masked_reg, RAM_SIZE - 1); | |
1529 | else | |
1530 | jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3); | |
1531 | ||
02487de7 | 1532 | if (!lut_is_32bit(state)) |
98fa08a5 | 1533 | jit_lshi(tmp, tmp, 1); |
cb72ea13 | 1534 | jit_add_state(tmp, tmp); |
d16005f8 PC |
1535 | |
1536 | /* Write NULL to the code LUT to invalidate any block that's there */ | |
02487de7 PC |
1537 | if (lut_is_32bit(state)) |
1538 | jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3); | |
1539 | else | |
1540 | jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3); | |
d16005f8 PC |
1541 | |
1542 | if (state->offset_ram != state->offset_scratch) { | |
1543 | jit_movi(tmp, state->offset_ram); | |
1544 | ||
22eee2ac | 1545 | to_end = jit_b(); |
9259d748 | 1546 | jit_patch(to_not_ram); |
d16005f8 PC |
1547 | } |
1548 | ||
d16005f8 PC |
1549 | if (state->offset_ram || state->offset_scratch) |
1550 | jit_movi(tmp, state->offset_scratch); | |
1551 | ||
1552 | if (state->offset_ram != state->offset_scratch) | |
1553 | jit_patch(to_end); | |
1554 | ||
1555 | if (state->offset_ram || state->offset_scratch) | |
1556 | jit_addr(tmp2, tmp2, tmp); | |
1557 | ||
1558 | lightrec_free_reg(reg_cache, tmp); | |
1559 | lightrec_free_reg(reg_cache, tmp3); | |
1560 | ||
9259d748 | 1561 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
02487de7 | 1562 | |
9259d748 | 1563 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1564 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
1565 | ||
1566 | jit_new_node_ww(swap_code, tmp, rt); | |
1567 | jit_new_node_www(code, 0, tmp2, tmp); | |
1568 | ||
1569 | lightrec_free_reg(reg_cache, tmp); | |
1570 | } else { | |
1571 | jit_new_node_www(code, 0, tmp2, rt); | |
1572 | } | |
d16005f8 PC |
1573 | |
1574 | lightrec_free_reg(reg_cache, rt); | |
1575 | lightrec_free_reg(reg_cache, tmp2); | |
1576 | } | |
1577 | ||
98fa08a5 | 1578 | static void rec_store(struct lightrec_cstate *state, |
02487de7 PC |
1579 | const struct block *block, u16 offset, |
1580 | jit_code_t code, jit_code_t swap_code) | |
d16005f8 | 1581 | { |
03535202 | 1582 | u32 flags = block->opcode_list[offset].flags; |
9259d748 | 1583 | u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags); |
03535202 | 1584 | bool no_invalidate = op_flag_no_invalidate(flags) || |
684432ad | 1585 | (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY); |
9259d748 PC |
1586 | union code c = block->opcode_list[offset].c; |
1587 | bool is_swc2 = c.i.op == OP_SWC2; | |
1588 | ||
1589 | if (is_swc2) { | |
1590 | switch (mode) { | |
1591 | case LIGHTREC_IO_RAM: | |
1592 | case LIGHTREC_IO_SCRATCH: | |
1593 | case LIGHTREC_IO_DIRECT: | |
1594 | case LIGHTREC_IO_DIRECT_HW: | |
cb72ea13 | 1595 | rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP); |
9259d748 PC |
1596 | break; |
1597 | default: | |
1598 | break; | |
1599 | } | |
1600 | } | |
22eee2ac | 1601 | |
9259d748 | 1602 | switch (mode) { |
22eee2ac | 1603 | case LIGHTREC_IO_RAM: |
02487de7 PC |
1604 | rec_store_ram(state, block, offset, code, |
1605 | swap_code, !no_invalidate); | |
22eee2ac PC |
1606 | break; |
1607 | case LIGHTREC_IO_SCRATCH: | |
02487de7 | 1608 | rec_store_scratch(state, block, offset, code, swap_code); |
22eee2ac PC |
1609 | break; |
1610 | case LIGHTREC_IO_DIRECT: | |
02487de7 PC |
1611 | if (no_invalidate) { |
1612 | rec_store_direct_no_invalidate(state, block, offset, | |
1613 | code, swap_code); | |
1614 | } else { | |
1615 | rec_store_direct(state, block, offset, code, swap_code); | |
1616 | } | |
22eee2ac | 1617 | break; |
ba3814c1 PC |
1618 | case LIGHTREC_IO_DIRECT_HW: |
1619 | rec_store_io(state, block, offset, code, swap_code); | |
1620 | break; | |
22eee2ac | 1621 | default: |
98fa08a5 | 1622 | rec_io(state, block, offset, true, false); |
9259d748 | 1623 | return; |
d16005f8 | 1624 | } |
9259d748 PC |
1625 | |
1626 | if (is_swc2) | |
cb72ea13 | 1627 | lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); |
d16005f8 PC |
1628 | } |
1629 | ||
98fa08a5 PC |
1630 | static void rec_SB(struct lightrec_cstate *state, |
1631 | const struct block *block, u16 offset) | |
d16005f8 PC |
1632 | { |
1633 | _jit_name(block->_jit, __func__); | |
02487de7 | 1634 | rec_store(state, block, offset, jit_code_stxi_c, 0); |
d16005f8 PC |
1635 | } |
1636 | ||
98fa08a5 PC |
1637 | static void rec_SH(struct lightrec_cstate *state, |
1638 | const struct block *block, u16 offset) | |
d16005f8 PC |
1639 | { |
1640 | _jit_name(block->_jit, __func__); | |
02487de7 PC |
1641 | rec_store(state, block, offset, |
1642 | jit_code_stxi_s, jit_code_bswapr_us); | |
d16005f8 PC |
1643 | } |
1644 | ||
98fa08a5 PC |
1645 | static void rec_SW(struct lightrec_cstate *state, |
1646 | const struct block *block, u16 offset) | |
1647 | ||
d16005f8 | 1648 | { |
9259d748 PC |
1649 | union code c = block->opcode_list[offset].c; |
1650 | ||
1651 | _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW"); | |
02487de7 PC |
1652 | rec_store(state, block, offset, |
1653 | jit_code_stxi_i, jit_code_bswapr_ui); | |
d16005f8 PC |
1654 | } |
1655 | ||
98fa08a5 PC |
1656 | static void rec_SWL(struct lightrec_cstate *state, |
1657 | const struct block *block, u16 offset) | |
d16005f8 PC |
1658 | { |
1659 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1660 | rec_io(state, block, offset, true, false); |
d16005f8 PC |
1661 | } |
1662 | ||
98fa08a5 PC |
1663 | static void rec_SWR(struct lightrec_cstate *state, |
1664 | const struct block *block, u16 offset) | |
d16005f8 PC |
1665 | { |
1666 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1667 | rec_io(state, block, offset, true, false); |
d16005f8 PC |
1668 | } |
1669 | ||
22eee2ac | 1670 | static void rec_load_memory(struct lightrec_cstate *cstate, |
02487de7 PC |
1671 | const struct block *block, u16 offset, |
1672 | jit_code_t code, jit_code_t swap_code, bool is_unsigned, | |
22eee2ac PC |
1673 | uintptr_t addr_offset, u32 addr_mask) |
1674 | { | |
1675 | struct regcache *reg_cache = cstate->reg_cache; | |
1676 | struct opcode *op = &block->opcode_list[offset]; | |
cb72ea13 | 1677 | bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; |
22eee2ac | 1678 | jit_state_t *_jit = block->_jit; |
9259d748 | 1679 | u8 rs, rt, out_reg, addr_reg, flags = REG_EXT; |
03535202 | 1680 | bool no_mask = op_flag_no_mask(op->flags); |
22eee2ac | 1681 | union code c = op->c; |
684432ad | 1682 | s8 reg_imm; |
02487de7 | 1683 | s16 imm; |
22eee2ac | 1684 | |
cb72ea13 PC |
1685 | if (load_delay || c.i.op == OP_LWC2) |
1686 | out_reg = REG_TEMP; | |
9259d748 PC |
1687 | else if (c.i.rt) |
1688 | out_reg = c.i.rt; | |
1689 | else | |
22eee2ac PC |
1690 | return; |
1691 | ||
1692 | if (is_unsigned) | |
1693 | flags |= REG_ZEXT; | |
1694 | ||
1695 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); | |
9259d748 | 1696 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
22eee2ac | 1697 | |
02487de7 PC |
1698 | if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) { |
1699 | jit_addi(rt, rs, (s16)c.i.imm); | |
22eee2ac | 1700 | addr_reg = rt; |
02487de7 | 1701 | imm = 0; |
22eee2ac PC |
1702 | } else { |
1703 | addr_reg = rs; | |
02487de7 PC |
1704 | imm = (s16)c.i.imm; |
1705 | } | |
1706 | ||
1707 | if (!no_mask) { | |
684432ad PC |
1708 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1709 | addr_mask); | |
1710 | ||
1711 | jit_andr(rt, addr_reg, reg_imm); | |
02487de7 | 1712 | addr_reg = rt; |
684432ad PC |
1713 | |
1714 | lightrec_free_reg(reg_cache, reg_imm); | |
22eee2ac PC |
1715 | } |
1716 | ||
1717 | if (addr_offset) { | |
684432ad PC |
1718 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1719 | addr_offset); | |
1720 | ||
1721 | jit_addr(rt, addr_reg, reg_imm); | |
22eee2ac | 1722 | addr_reg = rt; |
684432ad PC |
1723 | |
1724 | lightrec_free_reg(reg_cache, reg_imm); | |
22eee2ac PC |
1725 | } |
1726 | ||
02487de7 PC |
1727 | jit_new_node_www(code, rt, addr_reg, imm); |
1728 | ||
1729 | if (is_big_endian() && swap_code) { | |
1730 | jit_new_node_ww(swap_code, rt, rt); | |
1731 | ||
1732 | if (c.i.op == OP_LH) | |
1733 | jit_extr_s(rt, rt); | |
1734 | else if (c.i.op == OP_LW && __WORDSIZE == 64) | |
1735 | jit_extr_i(rt, rt); | |
1736 | } | |
22eee2ac PC |
1737 | |
1738 | lightrec_free_reg(reg_cache, rs); | |
1739 | lightrec_free_reg(reg_cache, rt); | |
1740 | } | |
1741 | ||
1742 | static void rec_load_ram(struct lightrec_cstate *cstate, | |
02487de7 PC |
1743 | const struct block *block, u16 offset, |
1744 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1745 | { |
1746 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1747 | ||
02487de7 PC |
1748 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
1749 | cstate->state->offset_ram, rec_ram_mask(cstate->state)); | |
22eee2ac PC |
1750 | } |
1751 | ||
1752 | static void rec_load_bios(struct lightrec_cstate *cstate, | |
02487de7 PC |
1753 | const struct block *block, u16 offset, |
1754 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1755 | { |
1756 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1757 | ||
02487de7 | 1758 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
22eee2ac PC |
1759 | cstate->state->offset_bios, 0x1fffffff); |
1760 | } | |
1761 | ||
1762 | static void rec_load_scratch(struct lightrec_cstate *cstate, | |
02487de7 PC |
1763 | const struct block *block, u16 offset, |
1764 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1765 | { |
1766 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1767 | ||
02487de7 | 1768 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
22eee2ac PC |
1769 | cstate->state->offset_scratch, 0x1fffffff); |
1770 | } | |
1771 | ||
ba3814c1 PC |
1772 | static void rec_load_io(struct lightrec_cstate *cstate, |
1773 | const struct block *block, u16 offset, | |
1774 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
1775 | { | |
1776 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1777 | ||
1778 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, | |
13b02197 | 1779 | cstate->state->offset_io, rec_io_mask(cstate->state)); |
ba3814c1 PC |
1780 | } |
1781 | ||
02487de7 PC |
1782 | static void rec_load_direct(struct lightrec_cstate *cstate, |
1783 | const struct block *block, u16 offset, | |
1784 | jit_code_t code, jit_code_t swap_code, | |
1785 | bool is_unsigned) | |
d16005f8 | 1786 | { |
98fa08a5 PC |
1787 | struct lightrec_state *state = cstate->state; |
1788 | struct regcache *reg_cache = cstate->reg_cache; | |
cb72ea13 PC |
1789 | struct opcode *op = &block->opcode_list[offset]; |
1790 | bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; | |
d16005f8 | 1791 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 1792 | jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2; |
9259d748 | 1793 | u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT; |
cb72ea13 | 1794 | union code c = op->c; |
684432ad PC |
1795 | s32 addr_mask; |
1796 | u32 reg_imm; | |
1797 | s8 offt_reg; | |
d16005f8 PC |
1798 | s16 imm; |
1799 | ||
cb72ea13 PC |
1800 | if (load_delay || c.i.op == OP_LWC2) |
1801 | out_reg = REG_TEMP; | |
9259d748 PC |
1802 | else if (c.i.rt) |
1803 | out_reg = c.i.rt; | |
1804 | else | |
d16005f8 PC |
1805 | return; |
1806 | ||
98fa08a5 PC |
1807 | if (is_unsigned) |
1808 | flags |= REG_ZEXT; | |
1809 | ||
d16005f8 | 1810 | jit_note(__FILE__, __LINE__); |
98fa08a5 | 1811 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
9259d748 | 1812 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
d16005f8 PC |
1813 | |
1814 | if ((state->offset_ram == state->offset_bios && | |
1815 | state->offset_ram == state->offset_scratch && | |
98fa08a5 | 1816 | state->mirrors_mapped) || !c.i.imm) { |
d16005f8 | 1817 | addr_reg = rs; |
98fa08a5 | 1818 | imm = (s16)c.i.imm; |
d16005f8 | 1819 | } else { |
98fa08a5 | 1820 | jit_addi(rt, rs, (s16)c.i.imm); |
d16005f8 PC |
1821 | addr_reg = rt; |
1822 | imm = 0; | |
1823 | ||
98fa08a5 | 1824 | if (c.i.rs != c.i.rt) |
d16005f8 PC |
1825 | lightrec_free_reg(reg_cache, rs); |
1826 | } | |
1827 | ||
1828 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
1829 | ||
1830 | if (state->offset_ram == state->offset_bios && | |
1831 | state->offset_ram == state->offset_scratch) { | |
684432ad PC |
1832 | if (!state->mirrors_mapped) |
1833 | addr_mask = 0x1f800000 | (RAM_SIZE - 1); | |
1834 | else | |
1835 | addr_mask = 0x1fffffff; | |
1836 | ||
1837 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, | |
1838 | addr_mask); | |
d16005f8 PC |
1839 | if (!state->mirrors_mapped) { |
1840 | jit_andi(tmp, addr_reg, BIT(28)); | |
1841 | jit_rshi_u(tmp, tmp, 28 - 22); | |
684432ad | 1842 | jit_orr(tmp, tmp, reg_imm); |
d16005f8 PC |
1843 | jit_andr(rt, addr_reg, tmp); |
1844 | } else { | |
684432ad | 1845 | jit_andr(rt, addr_reg, reg_imm); |
d16005f8 PC |
1846 | } |
1847 | ||
684432ad PC |
1848 | lightrec_free_reg(reg_cache, reg_imm); |
1849 | ||
1850 | if (state->offset_ram) { | |
1851 | offt_reg = lightrec_get_reg_with_value(reg_cache, | |
1852 | state->offset_ram); | |
1853 | if (offt_reg < 0) { | |
1854 | jit_movi(tmp, state->offset_ram); | |
1855 | lightrec_temp_set_value(reg_cache, tmp, | |
1856 | state->offset_ram); | |
1857 | } else { | |
1858 | lightrec_free_reg(reg_cache, tmp); | |
1859 | tmp = offt_reg; | |
1860 | } | |
1861 | } | |
d16005f8 PC |
1862 | } else { |
1863 | to_not_ram = jit_bmsi(addr_reg, BIT(28)); | |
1864 | ||
1865 | /* Convert to KUNSEG and avoid RAM mirrors */ | |
1866 | jit_andi(rt, addr_reg, RAM_SIZE - 1); | |
1867 | ||
1868 | if (state->offset_ram) | |
1869 | jit_movi(tmp, state->offset_ram); | |
1870 | ||
22eee2ac | 1871 | to_end = jit_b(); |
d16005f8 PC |
1872 | |
1873 | jit_patch(to_not_ram); | |
1874 | ||
1875 | if (state->offset_bios != state->offset_scratch) | |
1876 | to_not_bios = jit_bmci(addr_reg, BIT(22)); | |
1877 | ||
1878 | /* Convert to KUNSEG */ | |
1879 | jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1)); | |
1880 | ||
1881 | jit_movi(tmp, state->offset_bios); | |
1882 | ||
1883 | if (state->offset_bios != state->offset_scratch) { | |
22eee2ac | 1884 | to_end2 = jit_b(); |
d16005f8 PC |
1885 | |
1886 | jit_patch(to_not_bios); | |
1887 | ||
1888 | /* Convert to KUNSEG */ | |
1889 | jit_andi(rt, addr_reg, 0x1f800fff); | |
1890 | ||
1891 | if (state->offset_scratch) | |
1892 | jit_movi(tmp, state->offset_scratch); | |
1893 | ||
1894 | jit_patch(to_end2); | |
1895 | } | |
1896 | ||
1897 | jit_patch(to_end); | |
1898 | } | |
1899 | ||
1900 | if (state->offset_ram || state->offset_bios || state->offset_scratch) | |
1901 | jit_addr(rt, rt, tmp); | |
1902 | ||
1903 | jit_new_node_www(code, rt, rt, imm); | |
1904 | ||
02487de7 PC |
1905 | if (is_big_endian() && swap_code) { |
1906 | jit_new_node_ww(swap_code, rt, rt); | |
1907 | ||
1908 | if (c.i.op == OP_LH) | |
1909 | jit_extr_s(rt, rt); | |
1910 | else if (c.i.op == OP_LW && __WORDSIZE == 64) | |
1911 | jit_extr_i(rt, rt); | |
1912 | } | |
1913 | ||
d16005f8 PC |
1914 | lightrec_free_reg(reg_cache, addr_reg); |
1915 | lightrec_free_reg(reg_cache, rt); | |
1916 | lightrec_free_reg(reg_cache, tmp); | |
1917 | } | |
1918 | ||
98fa08a5 | 1919 | static void rec_load(struct lightrec_cstate *state, const struct block *block, |
02487de7 PC |
1920 | u16 offset, jit_code_t code, jit_code_t swap_code, |
1921 | bool is_unsigned) | |
d16005f8 | 1922 | { |
9259d748 PC |
1923 | const struct opcode *op = &block->opcode_list[offset]; |
1924 | u32 flags = op->flags; | |
98fa08a5 | 1925 | |
22eee2ac PC |
1926 | switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { |
1927 | case LIGHTREC_IO_RAM: | |
02487de7 | 1928 | rec_load_ram(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1929 | break; |
1930 | case LIGHTREC_IO_BIOS: | |
02487de7 | 1931 | rec_load_bios(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1932 | break; |
1933 | case LIGHTREC_IO_SCRATCH: | |
02487de7 | 1934 | rec_load_scratch(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac | 1935 | break; |
ba3814c1 PC |
1936 | case LIGHTREC_IO_DIRECT_HW: |
1937 | rec_load_io(state, block, offset, code, swap_code, is_unsigned); | |
1938 | break; | |
22eee2ac | 1939 | case LIGHTREC_IO_DIRECT: |
02487de7 | 1940 | rec_load_direct(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1941 | break; |
1942 | default: | |
98fa08a5 | 1943 | rec_io(state, block, offset, false, true); |
9259d748 PC |
1944 | return; |
1945 | } | |
1946 | ||
1947 | if (op->i.op == OP_LWC2) { | |
cb72ea13 PC |
1948 | rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP); |
1949 | lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); | |
22eee2ac | 1950 | } |
d16005f8 PC |
1951 | } |
1952 | ||
98fa08a5 | 1953 | static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1954 | { |
1955 | _jit_name(block->_jit, __func__); | |
02487de7 | 1956 | rec_load(state, block, offset, jit_code_ldxi_c, 0, false); |
d16005f8 PC |
1957 | } |
1958 | ||
98fa08a5 | 1959 | static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1960 | { |
1961 | _jit_name(block->_jit, __func__); | |
02487de7 | 1962 | rec_load(state, block, offset, jit_code_ldxi_uc, 0, true); |
d16005f8 PC |
1963 | } |
1964 | ||
98fa08a5 | 1965 | static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 1966 | { |
ba3814c1 PC |
1967 | jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s; |
1968 | ||
d16005f8 | 1969 | _jit_name(block->_jit, __func__); |
ba3814c1 | 1970 | rec_load(state, block, offset, code, jit_code_bswapr_us, false); |
d16005f8 PC |
1971 | } |
1972 | ||
98fa08a5 | 1973 | static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1974 | { |
1975 | _jit_name(block->_jit, __func__); | |
02487de7 | 1976 | rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true); |
d16005f8 PC |
1977 | } |
1978 | ||
98fa08a5 | 1979 | static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1980 | { |
1981 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1982 | rec_io(state, block, offset, true, true); |
d16005f8 PC |
1983 | } |
1984 | ||
98fa08a5 | 1985 | static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1986 | { |
1987 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1988 | rec_io(state, block, offset, true, true); |
d16005f8 PC |
1989 | } |
1990 | ||
98fa08a5 | 1991 | static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 1992 | { |
9259d748 | 1993 | union code c = block->opcode_list[offset].c; |
ba3814c1 PC |
1994 | jit_code_t code; |
1995 | ||
1996 | if (is_big_endian() && __WORDSIZE == 64) | |
1997 | code = jit_code_ldxi_ui; | |
1998 | else | |
1999 | code = jit_code_ldxi_i; | |
2000 | ||
9259d748 | 2001 | _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW"); |
ba3814c1 | 2002 | rec_load(state, block, offset, code, jit_code_bswapr_ui, false); |
d16005f8 PC |
2003 | } |
2004 | ||
e26c79a8 PC |
2005 | static void rec_exit_early(struct lightrec_cstate *state, |
2006 | const struct block *block, u16 offset, | |
2007 | u32 exit_code, u32 pc) | |
d16005f8 | 2008 | { |
ba3814c1 PC |
2009 | struct regcache *reg_cache = state->reg_cache; |
2010 | jit_state_t *_jit = block->_jit; | |
2011 | u8 tmp; | |
2012 | ||
98fa08a5 | 2013 | _jit_note(block->_jit, __FILE__, __LINE__); |
d16005f8 | 2014 | |
ba3814c1 PC |
2015 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
2016 | ||
2017 | jit_movi(tmp, exit_code); | |
2018 | jit_stxi_i(offsetof(struct lightrec_state, exit_flags), | |
2019 | LIGHTREC_REG_STATE, tmp); | |
2020 | ||
cb72ea13 PC |
2021 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, |
2022 | offsetof(struct lightrec_state, target_cycle)); | |
2023 | jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE); | |
2024 | jit_movi(LIGHTREC_REG_CYCLE, 0); | |
2025 | jit_stxi_i(offsetof(struct lightrec_state, target_cycle), | |
2026 | LIGHTREC_REG_STATE, tmp); | |
2027 | jit_stxi_i(offsetof(struct lightrec_state, current_cycle), | |
2028 | LIGHTREC_REG_STATE, tmp); | |
2029 | ||
ba3814c1 | 2030 | lightrec_free_reg(reg_cache, tmp); |
d16005f8 | 2031 | |
e26c79a8 | 2032 | lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true); |
d16005f8 PC |
2033 | } |
2034 | ||
98fa08a5 PC |
2035 | static void rec_special_SYSCALL(struct lightrec_cstate *state, |
2036 | const struct block *block, u16 offset) | |
d16005f8 PC |
2037 | { |
2038 | _jit_name(block->_jit, __func__); | |
e26c79a8 PC |
2039 | |
2040 | /* TODO: the return address should be "pc - 4" if we're a delay slot */ | |
2041 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL, | |
2042 | get_ds_pc(block, offset, 0)); | |
d16005f8 PC |
2043 | } |
2044 | ||
98fa08a5 PC |
2045 | static void rec_special_BREAK(struct lightrec_cstate *state, |
2046 | const struct block *block, u16 offset) | |
d16005f8 PC |
2047 | { |
2048 | _jit_name(block->_jit, __func__); | |
e26c79a8 PC |
2049 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK, |
2050 | get_ds_pc(block, offset, 0)); | |
d16005f8 PC |
2051 | } |
2052 | ||
fdf33147 PC |
2053 | static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset) |
2054 | { | |
2055 | struct regcache *reg_cache = state->reg_cache; | |
2056 | union code c = block->opcode_list[offset].c; | |
2057 | jit_state_t *_jit = block->_jit; | |
2058 | ||
2059 | jit_note(__FILE__, __LINE__); | |
9259d748 PC |
2060 | |
2061 | if (c.i.op != OP_SWC2) | |
2062 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); | |
fdf33147 PC |
2063 | |
2064 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC); | |
2065 | } | |
2066 | ||
fd58fa32 | 2067 | static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset) |
98fa08a5 PC |
2068 | { |
2069 | struct regcache *reg_cache = state->reg_cache; | |
2070 | union code c = block->opcode_list[offset].c; | |
2071 | jit_state_t *_jit = block->_jit; | |
d16005f8 | 2072 | |
98fa08a5 PC |
2073 | jit_note(__FILE__, __LINE__); |
2074 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); | |
2075 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); | |
cb72ea13 | 2076 | lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false); |
d16005f8 | 2077 | |
ba3814c1 | 2078 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC); |
d16005f8 | 2079 | |
98fa08a5 | 2080 | if (c.i.op == OP_CP0 && |
03535202 | 2081 | !op_flag_no_ds(block->opcode_list[offset].flags) && |
98fa08a5 PC |
2082 | (c.r.rd == 12 || c.r.rd == 13)) |
2083 | lightrec_emit_end_of_block(state, block, offset, -1, | |
2084 | get_ds_pc(block, offset, 1), | |
2085 | 0, 0, true); | |
d16005f8 PC |
2086 | } |
2087 | ||
98fa08a5 PC |
2088 | static void |
2089 | rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
d16005f8 | 2090 | { |
d16005f8 | 2091 | struct regcache *reg_cache = state->reg_cache; |
98fa08a5 | 2092 | union code c = block->opcode_list[offset].c; |
d16005f8 | 2093 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 2094 | u8 rt; |
d16005f8 PC |
2095 | |
2096 | jit_note(__FILE__, __LINE__); | |
2097 | ||
98fa08a5 | 2098 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT); |
d16005f8 | 2099 | |
98fa08a5 PC |
2100 | jit_ldxi_i(rt, LIGHTREC_REG_STATE, |
2101 | offsetof(struct lightrec_state, regs.cp0[c.r.rd])); | |
d16005f8 | 2102 | |
98fa08a5 PC |
2103 | lightrec_free_reg(reg_cache, rt); |
2104 | } | |
d16005f8 | 2105 | |
cb72ea13 PC |
2106 | static bool block_uses_icache(const struct lightrec_cstate *state, |
2107 | const struct block *block) | |
98fa08a5 | 2108 | { |
cb72ea13 | 2109 | const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM]; |
98fa08a5 | 2110 | u32 pc = kunseg(block->pc); |
d16005f8 | 2111 | |
cb72ea13 PC |
2112 | if (pc < map->pc || pc >= map->pc + map->length) |
2113 | return false; | |
2114 | ||
2115 | return (block->pc >> 28) < 0xa; | |
d16005f8 PC |
2116 | } |
2117 | ||
98fa08a5 PC |
2118 | static void |
2119 | rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
d16005f8 | 2120 | { |
98fa08a5 PC |
2121 | struct regcache *reg_cache = state->reg_cache; |
2122 | const union code c = block->opcode_list[offset].c; | |
2123 | jit_state_t *_jit = block->_jit; | |
fd58fa32 | 2124 | u8 rt, tmp = 0, tmp2, status; |
0cf41071 | 2125 | jit_node_t *to_end; |
98fa08a5 PC |
2126 | |
2127 | jit_note(__FILE__, __LINE__); | |
2128 | ||
2129 | switch(c.r.rd) { | |
2130 | case 1: | |
2131 | case 4: | |
2132 | case 8: | |
2133 | case 14: | |
2134 | case 15: | |
2135 | /* Those registers are read-only */ | |
2136 | return; | |
2137 | default: | |
2138 | break; | |
2139 | } | |
2140 | ||
cb72ea13 PC |
2141 | if (!block_uses_icache(state, block) && c.r.rd == 12) { |
2142 | /* If we are not running code from the RAM through kuseg or | |
2143 | * kseg0, handle writes to the Status register in C; as the | |
2144 | * code may toggle bit 16 which isolates the cache. Code | |
2145 | * running from kuseg or kseg0 in RAM cannot do that. */ | |
98fa08a5 PC |
2146 | rec_mtc(state, block, offset); |
2147 | return; | |
2148 | } | |
2149 | ||
2150 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); | |
2151 | ||
2152 | if (c.r.rd != 13) { | |
2153 | jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]), | |
2154 | LIGHTREC_REG_STATE, rt); | |
2155 | } | |
2156 | ||
2157 | if (c.r.rd == 12 || c.r.rd == 13) { | |
2158 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2159 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, | |
2160 | offsetof(struct lightrec_state, regs.cp0[13])); | |
fd58fa32 PC |
2161 | |
2162 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
98fa08a5 PC |
2163 | } |
2164 | ||
2165 | if (c.r.rd == 12) { | |
2166 | status = rt; | |
2167 | } else if (c.r.rd == 13) { | |
98fa08a5 PC |
2168 | /* Cause = (Cause & ~0x0300) | (value & 0x0300) */ |
2169 | jit_andi(tmp2, rt, 0x0300); | |
2170 | jit_ori(tmp, tmp, 0x0300); | |
2171 | jit_xori(tmp, tmp, 0x0300); | |
2172 | jit_orr(tmp, tmp, tmp2); | |
2173 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, | |
2174 | offsetof(struct lightrec_state, regs.cp0[12])); | |
2175 | jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]), | |
2176 | LIGHTREC_REG_STATE, tmp); | |
2177 | status = tmp2; | |
2178 | } | |
2179 | ||
2180 | if (c.r.rd == 12 || c.r.rd == 13) { | |
2181 | /* Exit dynarec in case there's a software interrupt. | |
2182 | * exit_flags = !!(status & tmp & 0x0300) & status; */ | |
2183 | jit_andr(tmp, tmp, status); | |
2184 | jit_andi(tmp, tmp, 0x0300); | |
2185 | jit_nei(tmp, tmp, 0); | |
2186 | jit_andr(tmp, tmp, status); | |
fd58fa32 PC |
2187 | } |
2188 | ||
2189 | if (c.r.rd == 12) { | |
2190 | /* Exit dynarec in case we unmask a hardware interrupt. | |
2191 | * exit_flags = !(~status & 0x401) */ | |
2192 | ||
2193 | jit_comr(tmp2, status); | |
2194 | jit_andi(tmp2, tmp2, 0x401); | |
2195 | jit_eqi(tmp2, tmp2, 0); | |
2196 | jit_orr(tmp, tmp, tmp2); | |
2197 | } | |
2198 | ||
0cf41071 PC |
2199 | lightrec_free_reg(reg_cache, rt); |
2200 | ||
fd58fa32 | 2201 | if (c.r.rd == 12 || c.r.rd == 13) { |
0cf41071 | 2202 | to_end = jit_beqi(tmp, 0); |
98fa08a5 | 2203 | |
0cf41071 PC |
2204 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, |
2205 | offsetof(struct lightrec_state, target_cycle)); | |
2206 | jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE); | |
2207 | jit_movi(LIGHTREC_REG_CYCLE, 0); | |
2208 | jit_stxi_i(offsetof(struct lightrec_state, target_cycle), | |
2209 | LIGHTREC_REG_STATE, tmp2); | |
2210 | jit_stxi_i(offsetof(struct lightrec_state, current_cycle), | |
2211 | LIGHTREC_REG_STATE, tmp2); | |
98fa08a5 | 2212 | |
0cf41071 PC |
2213 | |
2214 | jit_patch(to_end); | |
2215 | } | |
98fa08a5 | 2216 | |
03535202 | 2217 | if (!op_flag_no_ds(block->opcode_list[offset].flags) && |
13b02197 | 2218 | (c.r.rd == 12 || c.r.rd == 13)) { |
684432ad | 2219 | state->cycles += lightrec_cycles_of_opcode(state->state, c); |
13b02197 PC |
2220 | lightrec_emit_eob(state, block, offset + 1); |
2221 | } | |
d16005f8 PC |
2222 | } |
2223 | ||
98fa08a5 PC |
2224 | static void rec_cp0_MFC0(struct lightrec_cstate *state, |
2225 | const struct block *block, u16 offset) | |
d16005f8 PC |
2226 | { |
2227 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2228 | rec_mfc0(state, block, offset); |
d16005f8 PC |
2229 | } |
2230 | ||
98fa08a5 PC |
2231 | static void rec_cp0_CFC0(struct lightrec_cstate *state, |
2232 | const struct block *block, u16 offset) | |
d16005f8 PC |
2233 | { |
2234 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2235 | rec_mfc0(state, block, offset); |
d16005f8 PC |
2236 | } |
2237 | ||
98fa08a5 PC |
2238 | static void rec_cp0_MTC0(struct lightrec_cstate *state, |
2239 | const struct block *block, u16 offset) | |
d16005f8 PC |
2240 | { |
2241 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2242 | rec_mtc0(state, block, offset); |
d16005f8 PC |
2243 | } |
2244 | ||
98fa08a5 PC |
2245 | static void rec_cp0_CTC0(struct lightrec_cstate *state, |
2246 | const struct block *block, u16 offset) | |
d16005f8 PC |
2247 | { |
2248 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2249 | rec_mtc0(state, block, offset); |
d16005f8 PC |
2250 | } |
2251 | ||
02487de7 PC |
2252 | static unsigned int cp2d_i_offset(u8 reg) |
2253 | { | |
2254 | return offsetof(struct lightrec_state, regs.cp2d[reg]); | |
2255 | } | |
2256 | ||
2257 | static unsigned int cp2d_s_offset(u8 reg) | |
2258 | { | |
2259 | return cp2d_i_offset(reg) + is_big_endian() * 2; | |
2260 | } | |
2261 | ||
2262 | static unsigned int cp2c_i_offset(u8 reg) | |
2263 | { | |
2264 | return offsetof(struct lightrec_state, regs.cp2c[reg]); | |
2265 | } | |
2266 | ||
2267 | static unsigned int cp2c_s_offset(u8 reg) | |
2268 | { | |
2269 | return cp2c_i_offset(reg) + is_big_endian() * 2; | |
2270 | } | |
2271 | ||
9259d748 PC |
2272 | static void rec_cp2_do_mfc2(struct lightrec_cstate *state, |
2273 | const struct block *block, u16 offset, | |
2274 | u8 reg, u8 out_reg) | |
d16005f8 | 2275 | { |
fd58fa32 | 2276 | struct regcache *reg_cache = state->reg_cache; |
fd58fa32 PC |
2277 | jit_state_t *_jit = block->_jit; |
2278 | const u32 zext_regs = 0x300f0080; | |
2279 | u8 rt, tmp, tmp2, tmp3, out, flags; | |
fd58fa32 PC |
2280 | unsigned int i; |
2281 | ||
d16005f8 | 2282 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2283 | |
fdf33147 PC |
2284 | if (state->state->ops.cop2_notify) { |
2285 | /* We must call cop2_notify, handle that in C. */ | |
2286 | rec_mfc(state, block, offset); | |
2287 | return; | |
2288 | } | |
2289 | ||
fd58fa32 | 2290 | flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT; |
9259d748 PC |
2291 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
2292 | ||
2293 | if (reg == 15) | |
2294 | reg = 14; | |
fd58fa32 PC |
2295 | |
2296 | switch (reg) { | |
2297 | case 1: | |
2298 | case 3: | |
2299 | case 5: | |
2300 | case 8: | |
2301 | case 9: | |
2302 | case 10: | |
2303 | case 11: | |
02487de7 | 2304 | jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); |
fd58fa32 PC |
2305 | break; |
2306 | case 7: | |
2307 | case 16: | |
2308 | case 17: | |
2309 | case 18: | |
2310 | case 19: | |
02487de7 | 2311 | jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); |
fd58fa32 PC |
2312 | break; |
2313 | case 28: | |
2314 | case 29: | |
2315 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2316 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2317 | tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2318 | ||
2319 | for (i = 0; i < 3; i++) { | |
2320 | out = i == 0 ? rt : tmp; | |
2321 | ||
02487de7 | 2322 | jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i)); |
fd58fa32 PC |
2323 | jit_movi(tmp2, 0x1f); |
2324 | jit_rshi(out, tmp, 7); | |
2325 | ||
2326 | jit_ltr(tmp3, tmp2, out); | |
2327 | jit_movnr(out, tmp2, tmp3); | |
2328 | ||
2329 | jit_gei(tmp2, out, 0); | |
2330 | jit_movzr(out, tmp2, tmp2); | |
2331 | ||
2332 | if (i > 0) { | |
2333 | jit_lshi(tmp, tmp, 5 * i); | |
2334 | jit_orr(rt, rt, tmp); | |
2335 | } | |
2336 | } | |
2337 | ||
2338 | ||
2339 | lightrec_free_reg(reg_cache, tmp); | |
2340 | lightrec_free_reg(reg_cache, tmp2); | |
2341 | lightrec_free_reg(reg_cache, tmp3); | |
2342 | break; | |
2343 | default: | |
02487de7 | 2344 | jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg)); |
fd58fa32 PC |
2345 | break; |
2346 | } | |
2347 | ||
2348 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2349 | } |
2350 | ||
9259d748 PC |
2351 | static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, |
2352 | const struct block *block, u16 offset) | |
2353 | { | |
2354 | const union code c = block->opcode_list[offset].c; | |
2355 | ||
2356 | rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt); | |
2357 | } | |
2358 | ||
98fa08a5 PC |
2359 | static void rec_cp2_basic_CFC2(struct lightrec_cstate *state, |
2360 | const struct block *block, u16 offset) | |
d16005f8 | 2361 | { |
fd58fa32 PC |
2362 | struct regcache *reg_cache = state->reg_cache; |
2363 | const union code c = block->opcode_list[offset].c; | |
2364 | jit_state_t *_jit = block->_jit; | |
2365 | u8 rt; | |
2366 | ||
d16005f8 | 2367 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2368 | |
fdf33147 PC |
2369 | if (state->state->ops.cop2_notify) { |
2370 | /* We must call cop2_notify, handle that in C. */ | |
2371 | rec_mfc(state, block, offset); | |
2372 | return; | |
2373 | } | |
2374 | ||
fd58fa32 PC |
2375 | switch (c.r.rd) { |
2376 | case 4: | |
2377 | case 12: | |
2378 | case 20: | |
2379 | case 26: | |
2380 | case 27: | |
2381 | case 29: | |
2382 | case 30: | |
2383 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT); | |
02487de7 | 2384 | jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd)); |
fd58fa32 PC |
2385 | break; |
2386 | default: | |
2387 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT); | |
0cf41071 | 2388 | jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd)); |
fd58fa32 PC |
2389 | break; |
2390 | } | |
2391 | ||
2392 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2393 | } |
2394 | ||
9259d748 PC |
2395 | static void rec_cp2_do_mtc2(struct lightrec_cstate *state, |
2396 | const struct block *block, u16 offset, | |
2397 | u8 reg, u8 in_reg) | |
d16005f8 | 2398 | { |
fd58fa32 | 2399 | struct regcache *reg_cache = state->reg_cache; |
fd58fa32 | 2400 | jit_state_t *_jit = block->_jit; |
fd58fa32 PC |
2401 | u8 rt, tmp, tmp2, flags = 0; |
2402 | ||
d16005f8 | 2403 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2404 | |
fdf33147 PC |
2405 | if (state->state->ops.cop2_notify) { |
2406 | /* We must call cop2_notify, handle that in C. */ | |
2407 | rec_mtc(state, block, offset); | |
2408 | return; | |
2409 | } | |
2410 | ||
9259d748 | 2411 | if (reg == 31) |
fd58fa32 PC |
2412 | return; |
2413 | ||
9259d748 | 2414 | if (reg == 30) |
fd58fa32 PC |
2415 | flags |= REG_EXT; |
2416 | ||
9259d748 | 2417 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags); |
fd58fa32 | 2418 | |
9259d748 | 2419 | switch (reg) { |
fd58fa32 PC |
2420 | case 15: |
2421 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
02487de7 | 2422 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13)); |
fd58fa32 PC |
2423 | |
2424 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
02487de7 | 2425 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14)); |
fd58fa32 | 2426 | |
02487de7 PC |
2427 | jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp); |
2428 | jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2); | |
2429 | jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt); | |
fd58fa32 PC |
2430 | |
2431 | lightrec_free_reg(reg_cache, tmp); | |
2432 | lightrec_free_reg(reg_cache, tmp2); | |
2433 | break; | |
2434 | case 28: | |
2435 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2436 | ||
2437 | jit_lshi(tmp, rt, 7); | |
2438 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2439 | jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2440 | |
2441 | jit_lshi(tmp, rt, 2); | |
2442 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2443 | jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2444 | |
2445 | jit_rshi(tmp, rt, 3); | |
2446 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2447 | jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2448 | |
2449 | lightrec_free_reg(reg_cache, tmp); | |
2450 | break; | |
2451 | case 30: | |
2452 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
fd58fa32 PC |
2453 | |
2454 | /* if (rt < 0) rt = ~rt; */ | |
2455 | jit_rshi(tmp, rt, 31); | |
2456 | jit_xorr(tmp, rt, tmp); | |
2457 | ||
cb72ea13 PC |
2458 | /* Count leading zeros */ |
2459 | jit_clzr(tmp, tmp); | |
2460 | if (__WORDSIZE != 32) | |
2461 | jit_subi(tmp, tmp, __WORDSIZE - 32); | |
fd58fa32 | 2462 | |
cb72ea13 | 2463 | jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2464 | |
2465 | lightrec_free_reg(reg_cache, tmp); | |
cb72ea13 | 2466 | fallthrough; |
fd58fa32 | 2467 | default: |
9259d748 | 2468 | jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2469 | break; |
2470 | } | |
2471 | ||
2472 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2473 | } |
2474 | ||
9259d748 PC |
2475 | static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, |
2476 | const struct block *block, u16 offset) | |
2477 | { | |
2478 | const union code c = block->opcode_list[offset].c; | |
2479 | ||
2480 | rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt); | |
2481 | } | |
2482 | ||
98fa08a5 PC |
2483 | static void rec_cp2_basic_CTC2(struct lightrec_cstate *state, |
2484 | const struct block *block, u16 offset) | |
d16005f8 | 2485 | { |
fd58fa32 PC |
2486 | struct regcache *reg_cache = state->reg_cache; |
2487 | const union code c = block->opcode_list[offset].c; | |
2488 | jit_state_t *_jit = block->_jit; | |
2489 | u8 rt, tmp, tmp2; | |
2490 | ||
98fa08a5 | 2491 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2492 | |
fdf33147 PC |
2493 | if (state->state->ops.cop2_notify) { |
2494 | /* We must call cop2_notify, handle that in C. */ | |
2495 | rec_mtc(state, block, offset); | |
2496 | return; | |
2497 | } | |
2498 | ||
fd58fa32 PC |
2499 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
2500 | ||
2501 | switch (c.r.rd) { | |
2502 | case 4: | |
2503 | case 12: | |
2504 | case 20: | |
2505 | case 26: | |
2506 | case 27: | |
2507 | case 29: | |
2508 | case 30: | |
02487de7 | 2509 | jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2510 | break; |
2511 | case 31: | |
2512 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2513 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2514 | ||
2515 | jit_andi(tmp, rt, 0x7f87e000); | |
2516 | jit_nei(tmp, tmp, 0); | |
2517 | jit_lshi(tmp, tmp, 31); | |
2518 | ||
2519 | jit_andi(tmp2, rt, 0x7ffff000); | |
2520 | jit_orr(tmp, tmp2, tmp); | |
2521 | ||
02487de7 | 2522 | jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2523 | |
2524 | lightrec_free_reg(reg_cache, tmp); | |
2525 | lightrec_free_reg(reg_cache, tmp2); | |
2526 | break; | |
2527 | ||
2528 | default: | |
02487de7 | 2529 | jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2530 | } |
2531 | ||
2532 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2533 | } |
2534 | ||
98fa08a5 PC |
2535 | static void rec_cp0_RFE(struct lightrec_cstate *state, |
2536 | const struct block *block, u16 offset) | |
d16005f8 | 2537 | { |
98fa08a5 | 2538 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 2539 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 2540 | u8 status, tmp; |
d16005f8 PC |
2541 | |
2542 | jit_name(__func__); | |
2543 | jit_note(__FILE__, __LINE__); | |
2544 | ||
98fa08a5 PC |
2545 | status = lightrec_alloc_reg_temp(reg_cache, _jit); |
2546 | jit_ldxi_i(status, LIGHTREC_REG_STATE, | |
2547 | offsetof(struct lightrec_state, regs.cp0[12])); | |
d16005f8 | 2548 | |
98fa08a5 | 2549 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
d16005f8 | 2550 | |
98fa08a5 PC |
2551 | /* status = ((status >> 2) & 0xf) | status & ~0xf; */ |
2552 | jit_rshi(tmp, status, 2); | |
2553 | jit_andi(tmp, tmp, 0xf); | |
2554 | jit_andi(status, status, ~0xful); | |
2555 | jit_orr(status, status, tmp); | |
2556 | ||
2557 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, | |
2558 | offsetof(struct lightrec_state, regs.cp0[13])); | |
2559 | jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]), | |
2560 | LIGHTREC_REG_STATE, status); | |
2561 | ||
2562 | /* Exit dynarec in case there's a software interrupt. | |
2563 | * exit_flags = !!(status & cause & 0x0300) & status; */ | |
2564 | jit_andr(tmp, tmp, status); | |
2565 | jit_andi(tmp, tmp, 0x0300); | |
2566 | jit_nei(tmp, tmp, 0); | |
2567 | jit_andr(tmp, tmp, status); | |
2568 | jit_stxi_i(offsetof(struct lightrec_state, exit_flags), | |
2569 | LIGHTREC_REG_STATE, tmp); | |
2570 | ||
2571 | lightrec_free_reg(reg_cache, status); | |
d16005f8 | 2572 | lightrec_free_reg(reg_cache, tmp); |
d16005f8 PC |
2573 | } |
2574 | ||
98fa08a5 PC |
2575 | static void rec_CP(struct lightrec_cstate *state, |
2576 | const struct block *block, u16 offset) | |
d16005f8 | 2577 | { |
98fa08a5 | 2578 | union code c = block->opcode_list[offset].c; |
d16005f8 PC |
2579 | jit_state_t *_jit = block->_jit; |
2580 | ||
2581 | jit_name(__func__); | |
2582 | jit_note(__FILE__, __LINE__); | |
2583 | ||
ba3814c1 | 2584 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP); |
d16005f8 PC |
2585 | } |
2586 | ||
98fa08a5 PC |
2587 | static void rec_meta_MOV(struct lightrec_cstate *state, |
2588 | const struct block *block, u16 offset) | |
d16005f8 | 2589 | { |
d16005f8 | 2590 | struct regcache *reg_cache = state->reg_cache; |
9259d748 PC |
2591 | const struct opcode *op = &block->opcode_list[offset]; |
2592 | union code c = op->c; | |
d16005f8 | 2593 | jit_state_t *_jit = block->_jit; |
9259d748 | 2594 | bool unload_rd; |
684432ad | 2595 | bool unload_rs, discard_rs; |
d16005f8 PC |
2596 | u8 rs, rd; |
2597 | ||
2598 | _jit_name(block->_jit, __func__); | |
2599 | jit_note(__FILE__, __LINE__); | |
9259d748 | 2600 | |
684432ad PC |
2601 | unload_rs = OPT_EARLY_UNLOAD |
2602 | && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD; | |
2603 | discard_rs = OPT_EARLY_UNLOAD | |
2604 | && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD; | |
2605 | ||
2606 | if ((unload_rs || discard_rs) && c.m.rs) { | |
2607 | /* If the source register is going to be unloaded or discarded, | |
2608 | * then we can simply mark its host register as now pointing to | |
2609 | * the destination register. */ | |
2610 | pr_debug("Remap %s to %s at offset 0x%x\n", | |
2611 | lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd), | |
2612 | offset << 2); | |
2613 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
2614 | lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs); | |
2615 | lightrec_free_reg(reg_cache, rs); | |
2616 | return; | |
2617 | } | |
2618 | ||
9259d748 PC |
2619 | unload_rd = OPT_EARLY_UNLOAD |
2620 | && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD; | |
2621 | ||
cb72ea13 PC |
2622 | if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) { |
2623 | /* The source register is not yet loaded - we can load its value | |
2624 | * from the register cache directly into the target register. */ | |
2625 | rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); | |
2626 | ||
2627 | jit_ldxi_i(rd, LIGHTREC_REG_STATE, | |
2628 | offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2)); | |
d16005f8 | 2629 | |
cb72ea13 PC |
2630 | lightrec_free_reg(reg_cache, rd); |
2631 | } else if (unload_rd) { | |
9259d748 PC |
2632 | /* If the destination register will be unloaded right after the |
2633 | * MOV meta-opcode, we don't actually need to write any host | |
2634 | * register - we can just store the source register directly to | |
2635 | * the register cache, at the offset corresponding to the | |
2636 | * destination register. */ | |
cb72ea13 PC |
2637 | lightrec_discard_reg_if_loaded(reg_cache, c.m.rd); |
2638 | ||
2639 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
9259d748 PC |
2640 | |
2641 | jit_stxi_i(offsetof(struct lightrec_state, regs.gpr) | |
cb72ea13 | 2642 | + (c.m.rd << 2), LIGHTREC_REG_STATE, rs); |
d16005f8 | 2643 | |
98fa08a5 | 2644 | lightrec_free_reg(reg_cache, rs); |
9259d748 | 2645 | } else { |
cb72ea13 PC |
2646 | if (c.m.rs) |
2647 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
2648 | ||
2649 | rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); | |
9259d748 | 2650 | |
cb72ea13 | 2651 | if (c.m.rs == 0) { |
9259d748 | 2652 | jit_movi(rd, 0); |
cb72ea13 | 2653 | } else { |
9259d748 | 2654 | jit_extr_i(rd, rs); |
cb72ea13 PC |
2655 | lightrec_free_reg(reg_cache, rs); |
2656 | } | |
9259d748 PC |
2657 | |
2658 | lightrec_free_reg(reg_cache, rd); | |
2659 | } | |
d16005f8 PC |
2660 | } |
2661 | ||
98fa08a5 PC |
2662 | static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state, |
2663 | const struct block *block, | |
2664 | u16 offset) | |
d16005f8 | 2665 | { |
98fa08a5 PC |
2666 | struct regcache *reg_cache = state->reg_cache; |
2667 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 2668 | jit_state_t *_jit = block->_jit; |
cb72ea13 | 2669 | u8 rs, rd; |
d16005f8 | 2670 | |
98fa08a5 | 2671 | _jit_name(block->_jit, __func__); |
d16005f8 PC |
2672 | jit_note(__FILE__, __LINE__); |
2673 | ||
684432ad PC |
2674 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
2675 | c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd); | |
d16005f8 | 2676 | |
cb72ea13 PC |
2677 | if (c.m.op == OP_META_EXTC) |
2678 | jit_extr_c(rd, rs); | |
98fa08a5 | 2679 | else |
cb72ea13 | 2680 | jit_extr_s(rd, rs); |
d16005f8 | 2681 | |
98fa08a5 | 2682 | lightrec_free_reg(reg_cache, rs); |
cb72ea13 | 2683 | lightrec_free_reg(reg_cache, rd); |
d16005f8 PC |
2684 | } |
2685 | ||
ba3814c1 PC |
2686 | static void rec_meta_MULT2(struct lightrec_cstate *state, |
2687 | const struct block *block, | |
2688 | u16 offset) | |
2689 | { | |
2690 | struct regcache *reg_cache = state->reg_cache; | |
2691 | union code c = block->opcode_list[offset].c; | |
2692 | jit_state_t *_jit = block->_jit; | |
2693 | u8 reg_lo = get_mult_div_lo(c); | |
2694 | u8 reg_hi = get_mult_div_hi(c); | |
2695 | u32 flags = block->opcode_list[offset].flags; | |
2696 | bool is_signed = c.i.op == OP_META_MULT2; | |
2697 | u8 rs, lo, hi, rflags = 0, hiflags = 0; | |
0cf41071 | 2698 | unsigned int i; |
ba3814c1 PC |
2699 | |
2700 | if (!op_flag_no_hi(flags) && c.r.op < 32) { | |
2701 | rflags = is_signed ? REG_EXT : REG_ZEXT; | |
2702 | hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT); | |
2703 | } | |
2704 | ||
2705 | _jit_name(block->_jit, __func__); | |
2706 | jit_note(__FILE__, __LINE__); | |
2707 | ||
2708 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags); | |
2709 | ||
0cf41071 PC |
2710 | /* |
2711 | * We must handle the case where one of the output registers is our rs | |
2712 | * input register. Thanksfully, computing LO/HI can be done in any | |
2713 | * order. Here, we make sure that the computation that overwrites the | |
2714 | * input register is always performed last. | |
2715 | */ | |
2716 | for (i = 0; i < 2; i++) { | |
2717 | if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) { | |
2718 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); | |
2719 | ||
2720 | if (c.r.op < 32) | |
2721 | jit_lshi(lo, rs, c.r.op); | |
2722 | else | |
2723 | jit_movi(lo, 0); | |
ba3814c1 | 2724 | |
0cf41071 PC |
2725 | lightrec_free_reg(reg_cache, lo); |
2726 | continue; | |
2727 | } | |
ba3814c1 | 2728 | |
0cf41071 PC |
2729 | if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) { |
2730 | hi = lightrec_alloc_reg_out(reg_cache, _jit, | |
2731 | reg_hi, hiflags); | |
ba3814c1 | 2732 | |
0cf41071 PC |
2733 | if (c.r.op >= 32) |
2734 | jit_lshi(hi, rs, c.r.op - 32); | |
2735 | else if (is_signed) | |
2736 | jit_rshi(hi, rs, 32 - c.r.op); | |
2737 | else | |
2738 | jit_rshi_u(hi, rs, 32 - c.r.op); | |
ba3814c1 | 2739 | |
0cf41071 PC |
2740 | lightrec_free_reg(reg_cache, hi); |
2741 | } | |
ba3814c1 PC |
2742 | } |
2743 | ||
2744 | lightrec_free_reg(reg_cache, rs); | |
2745 | ||
2746 | _jit_name(block->_jit, __func__); | |
2747 | jit_note(__FILE__, __LINE__); | |
2748 | } | |
2749 | ||
cb72ea13 PC |
2750 | static void rec_meta_COM(struct lightrec_cstate *state, |
2751 | const struct block *block, u16 offset) | |
2752 | { | |
2753 | struct regcache *reg_cache = state->reg_cache; | |
2754 | union code c = block->opcode_list[offset].c; | |
2755 | jit_state_t *_jit = block->_jit; | |
2756 | u8 rd, rs, flags; | |
2757 | ||
2758 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
2759 | |
2760 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
2761 | c.m.rs, c.m.rd, 0, 0, &rs, &rd); | |
cb72ea13 PC |
2762 | |
2763 | flags = lightrec_get_reg_in_flags(reg_cache, rs); | |
2764 | ||
2765 | lightrec_set_reg_out_flags(reg_cache, rd, | |
2766 | flags & REG_EXT); | |
2767 | ||
2768 | jit_comr(rd, rs); | |
2769 | ||
2770 | lightrec_free_reg(reg_cache, rs); | |
2771 | lightrec_free_reg(reg_cache, rd); | |
2772 | } | |
2773 | ||
e26c79a8 PC |
2774 | static void unknown_opcode(struct lightrec_cstate *state, |
2775 | const struct block *block, u16 offset) | |
2776 | { | |
2777 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP, | |
2778 | block->pc + (offset << 2)); | |
2779 | } | |
2780 | ||
d16005f8 | 2781 | static const lightrec_rec_func_t rec_standard[64] = { |
98fa08a5 | 2782 | SET_DEFAULT_ELM(rec_standard, unknown_opcode), |
d16005f8 PC |
2783 | [OP_SPECIAL] = rec_SPECIAL, |
2784 | [OP_REGIMM] = rec_REGIMM, | |
2785 | [OP_J] = rec_J, | |
2786 | [OP_JAL] = rec_JAL, | |
2787 | [OP_BEQ] = rec_BEQ, | |
2788 | [OP_BNE] = rec_BNE, | |
2789 | [OP_BLEZ] = rec_BLEZ, | |
2790 | [OP_BGTZ] = rec_BGTZ, | |
2791 | [OP_ADDI] = rec_ADDI, | |
2792 | [OP_ADDIU] = rec_ADDIU, | |
2793 | [OP_SLTI] = rec_SLTI, | |
2794 | [OP_SLTIU] = rec_SLTIU, | |
2795 | [OP_ANDI] = rec_ANDI, | |
2796 | [OP_ORI] = rec_ORI, | |
2797 | [OP_XORI] = rec_XORI, | |
2798 | [OP_LUI] = rec_LUI, | |
2799 | [OP_CP0] = rec_CP0, | |
2800 | [OP_CP2] = rec_CP2, | |
2801 | [OP_LB] = rec_LB, | |
2802 | [OP_LH] = rec_LH, | |
2803 | [OP_LWL] = rec_LWL, | |
2804 | [OP_LW] = rec_LW, | |
2805 | [OP_LBU] = rec_LBU, | |
2806 | [OP_LHU] = rec_LHU, | |
2807 | [OP_LWR] = rec_LWR, | |
2808 | [OP_SB] = rec_SB, | |
2809 | [OP_SH] = rec_SH, | |
2810 | [OP_SWL] = rec_SWL, | |
2811 | [OP_SW] = rec_SW, | |
2812 | [OP_SWR] = rec_SWR, | |
9259d748 PC |
2813 | [OP_LWC2] = rec_LW, |
2814 | [OP_SWC2] = rec_SW, | |
d16005f8 | 2815 | |
cb72ea13 | 2816 | [OP_META] = rec_META, |
ba3814c1 PC |
2817 | [OP_META_MULT2] = rec_meta_MULT2, |
2818 | [OP_META_MULTU2] = rec_meta_MULT2, | |
d16005f8 PC |
2819 | }; |
2820 | ||
2821 | static const lightrec_rec_func_t rec_special[64] = { | |
98fa08a5 | 2822 | SET_DEFAULT_ELM(rec_special, unknown_opcode), |
d16005f8 PC |
2823 | [OP_SPECIAL_SLL] = rec_special_SLL, |
2824 | [OP_SPECIAL_SRL] = rec_special_SRL, | |
2825 | [OP_SPECIAL_SRA] = rec_special_SRA, | |
2826 | [OP_SPECIAL_SLLV] = rec_special_SLLV, | |
2827 | [OP_SPECIAL_SRLV] = rec_special_SRLV, | |
2828 | [OP_SPECIAL_SRAV] = rec_special_SRAV, | |
2829 | [OP_SPECIAL_JR] = rec_special_JR, | |
2830 | [OP_SPECIAL_JALR] = rec_special_JALR, | |
2831 | [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL, | |
2832 | [OP_SPECIAL_BREAK] = rec_special_BREAK, | |
2833 | [OP_SPECIAL_MFHI] = rec_special_MFHI, | |
2834 | [OP_SPECIAL_MTHI] = rec_special_MTHI, | |
2835 | [OP_SPECIAL_MFLO] = rec_special_MFLO, | |
2836 | [OP_SPECIAL_MTLO] = rec_special_MTLO, | |
2837 | [OP_SPECIAL_MULT] = rec_special_MULT, | |
2838 | [OP_SPECIAL_MULTU] = rec_special_MULTU, | |
2839 | [OP_SPECIAL_DIV] = rec_special_DIV, | |
2840 | [OP_SPECIAL_DIVU] = rec_special_DIVU, | |
2841 | [OP_SPECIAL_ADD] = rec_special_ADD, | |
2842 | [OP_SPECIAL_ADDU] = rec_special_ADDU, | |
2843 | [OP_SPECIAL_SUB] = rec_special_SUB, | |
2844 | [OP_SPECIAL_SUBU] = rec_special_SUBU, | |
2845 | [OP_SPECIAL_AND] = rec_special_AND, | |
2846 | [OP_SPECIAL_OR] = rec_special_OR, | |
2847 | [OP_SPECIAL_XOR] = rec_special_XOR, | |
2848 | [OP_SPECIAL_NOR] = rec_special_NOR, | |
2849 | [OP_SPECIAL_SLT] = rec_special_SLT, | |
2850 | [OP_SPECIAL_SLTU] = rec_special_SLTU, | |
2851 | }; | |
2852 | ||
2853 | static const lightrec_rec_func_t rec_regimm[64] = { | |
98fa08a5 | 2854 | SET_DEFAULT_ELM(rec_regimm, unknown_opcode), |
d16005f8 PC |
2855 | [OP_REGIMM_BLTZ] = rec_regimm_BLTZ, |
2856 | [OP_REGIMM_BGEZ] = rec_regimm_BGEZ, | |
2857 | [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL, | |
2858 | [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL, | |
2859 | }; | |
2860 | ||
2861 | static const lightrec_rec_func_t rec_cp0[64] = { | |
98fa08a5 | 2862 | SET_DEFAULT_ELM(rec_cp0, rec_CP), |
d16005f8 PC |
2863 | [OP_CP0_MFC0] = rec_cp0_MFC0, |
2864 | [OP_CP0_CFC0] = rec_cp0_CFC0, | |
2865 | [OP_CP0_MTC0] = rec_cp0_MTC0, | |
2866 | [OP_CP0_CTC0] = rec_cp0_CTC0, | |
2867 | [OP_CP0_RFE] = rec_cp0_RFE, | |
2868 | }; | |
2869 | ||
2870 | static const lightrec_rec_func_t rec_cp2_basic[64] = { | |
98fa08a5 | 2871 | SET_DEFAULT_ELM(rec_cp2_basic, rec_CP), |
d16005f8 PC |
2872 | [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2, |
2873 | [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2, | |
2874 | [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2, | |
2875 | [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2, | |
2876 | }; | |
2877 | ||
cb72ea13 PC |
2878 | static const lightrec_rec_func_t rec_meta[64] = { |
2879 | SET_DEFAULT_ELM(rec_meta, unknown_opcode), | |
2880 | [OP_META_MOV] = rec_meta_MOV, | |
2881 | [OP_META_EXTC] = rec_meta_EXTC_EXTS, | |
2882 | [OP_META_EXTS] = rec_meta_EXTC_EXTS, | |
2883 | [OP_META_COM] = rec_meta_COM, | |
2884 | }; | |
2885 | ||
98fa08a5 PC |
2886 | static void rec_SPECIAL(struct lightrec_cstate *state, |
2887 | const struct block *block, u16 offset) | |
d16005f8 | 2888 | { |
98fa08a5 PC |
2889 | union code c = block->opcode_list[offset].c; |
2890 | lightrec_rec_func_t f = rec_special[c.r.op]; | |
2891 | ||
2892 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2893 | unknown_opcode(state, block, offset); | |
d16005f8 | 2894 | else |
98fa08a5 | 2895 | (*f)(state, block, offset); |
d16005f8 PC |
2896 | } |
2897 | ||
98fa08a5 PC |
2898 | static void rec_REGIMM(struct lightrec_cstate *state, |
2899 | const struct block *block, u16 offset) | |
d16005f8 | 2900 | { |
98fa08a5 PC |
2901 | union code c = block->opcode_list[offset].c; |
2902 | lightrec_rec_func_t f = rec_regimm[c.r.rt]; | |
2903 | ||
2904 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2905 | unknown_opcode(state, block, offset); | |
d16005f8 | 2906 | else |
98fa08a5 | 2907 | (*f)(state, block, offset); |
d16005f8 PC |
2908 | } |
2909 | ||
98fa08a5 PC |
2910 | static void rec_CP0(struct lightrec_cstate *state, |
2911 | const struct block *block, u16 offset) | |
d16005f8 | 2912 | { |
98fa08a5 PC |
2913 | union code c = block->opcode_list[offset].c; |
2914 | lightrec_rec_func_t f = rec_cp0[c.r.rs]; | |
2915 | ||
2916 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2917 | rec_CP(state, block, offset); | |
d16005f8 | 2918 | else |
98fa08a5 | 2919 | (*f)(state, block, offset); |
d16005f8 PC |
2920 | } |
2921 | ||
98fa08a5 PC |
2922 | static void rec_CP2(struct lightrec_cstate *state, |
2923 | const struct block *block, u16 offset) | |
d16005f8 | 2924 | { |
98fa08a5 PC |
2925 | union code c = block->opcode_list[offset].c; |
2926 | ||
2927 | if (c.r.op == OP_CP2_BASIC) { | |
2928 | lightrec_rec_func_t f = rec_cp2_basic[c.r.rs]; | |
2929 | ||
2930 | if (HAS_DEFAULT_ELM || likely(f)) { | |
2931 | (*f)(state, block, offset); | |
d16005f8 PC |
2932 | return; |
2933 | } | |
2934 | } | |
2935 | ||
98fa08a5 | 2936 | rec_CP(state, block, offset); |
d16005f8 PC |
2937 | } |
2938 | ||
cb72ea13 PC |
2939 | static void rec_META(struct lightrec_cstate *state, |
2940 | const struct block *block, u16 offset) | |
2941 | { | |
2942 | union code c = block->opcode_list[offset].c; | |
2943 | lightrec_rec_func_t f = rec_meta[c.m.op]; | |
2944 | ||
2945 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2946 | unknown_opcode(state, block, offset); | |
2947 | else | |
2948 | (*f)(state, block, offset); | |
2949 | } | |
2950 | ||
98fa08a5 PC |
2951 | void lightrec_rec_opcode(struct lightrec_cstate *state, |
2952 | const struct block *block, u16 offset) | |
d16005f8 | 2953 | { |
98fa08a5 PC |
2954 | struct regcache *reg_cache = state->reg_cache; |
2955 | struct lightrec_branch_target *target; | |
2956 | const struct opcode *op = &block->opcode_list[offset]; | |
2957 | jit_state_t *_jit = block->_jit; | |
2958 | lightrec_rec_func_t f; | |
03535202 | 2959 | u16 unload_offset; |
98fa08a5 | 2960 | |
03535202 PC |
2961 | if (op_flag_sync(op->flags)) { |
2962 | if (state->cycles) | |
2963 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); | |
98fa08a5 PC |
2964 | state->cycles = 0; |
2965 | ||
2966 | lightrec_storeback_regs(reg_cache, _jit); | |
2967 | lightrec_regcache_reset(reg_cache); | |
2968 | ||
2969 | pr_debug("Adding branch target at offset 0x%x\n", offset << 2); | |
2970 | target = &state->targets[state->nb_targets++]; | |
2971 | target->offset = offset; | |
2972 | target->label = jit_indirect(); | |
2973 | } | |
2974 | ||
2975 | if (likely(op->opcode)) { | |
2976 | f = rec_standard[op->i.op]; | |
2977 | ||
2978 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2979 | unknown_opcode(state, block, offset); | |
2980 | else | |
2981 | (*f)(state, block, offset); | |
2982 | } | |
2983 | ||
03535202 PC |
2984 | if (OPT_EARLY_UNLOAD) { |
2985 | unload_offset = offset + | |
2986 | (has_delay_slot(op->c) && !op_flag_no_ds(op->flags)); | |
2987 | ||
2988 | lightrec_do_early_unload(state, block, unload_offset); | |
98fa08a5 | 2989 | } |
cb72ea13 PC |
2990 | |
2991 | state->no_load_delay = false; | |
d16005f8 | 2992 | } |