Commit | Line | Data |
---|---|---|
98fa08a5 | 1 | // SPDX-License-Identifier: LGPL-2.1-or-later |
d16005f8 | 2 | /* |
98fa08a5 | 3 | * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net> |
d16005f8 PC |
4 | */ |
5 | ||
6 | #include "blockcache.h" | |
7 | #include "debug.h" | |
8 | #include "disassembler.h" | |
9 | #include "emitter.h" | |
98fa08a5 | 10 | #include "lightning-wrapper.h" |
d16005f8 PC |
11 | #include "optimizer.h" |
12 | #include "regcache.h" | |
13 | ||
d16005f8 PC |
14 | #include <stdbool.h> |
15 | #include <stddef.h> | |
16 | ||
5459088b PC |
17 | #define LIGHTNING_UNALIGNED_32BIT 4 |
18 | ||
98fa08a5 | 19 | typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16); |
d16005f8 PC |
20 | |
21 | /* Forward declarations */ | |
98fa08a5 PC |
22 | static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset); |
23 | static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
24 | static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
25 | static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset); | |
cb72ea13 | 26 | static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset); |
9259d748 PC |
27 | static void rec_cp2_do_mtc2(struct lightrec_cstate *state, |
28 | const struct block *block, u16 offset, u8 reg, u8 in_reg); | |
29 | static void rec_cp2_do_mfc2(struct lightrec_cstate *state, | |
30 | const struct block *block, u16 offset, | |
31 | u8 reg, u8 out_reg); | |
d16005f8 | 32 | |
ba3814c1 | 33 | static void |
cb72ea13 | 34 | lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void)) |
ba3814c1 PC |
35 | { |
36 | /* Prevent jit_jmpi() from using our cycles register as a temporary */ | |
37 | jit_live(LIGHTREC_REG_CYCLE); | |
38 | ||
cb72ea13 PC |
39 | jit_patch_abs(jit_jmpi(), fn); |
40 | } | |
41 | ||
42 | static void | |
43 | lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit) | |
44 | { | |
45 | lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func); | |
46 | } | |
47 | ||
48 | static void | |
49 | lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit) | |
50 | { | |
51 | lightrec_jump_to_fn(_jit, state->state->ds_check_func); | |
ba3814c1 PC |
52 | } |
53 | ||
9259d748 PC |
54 | static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit, |
55 | u8 ra_reg, u32 pc, u32 link) | |
56 | { | |
57 | u8 link_reg; | |
58 | ||
59 | link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0); | |
60 | lightrec_load_imm(reg_cache, _jit, link_reg, pc, link); | |
61 | lightrec_free_reg(reg_cache, link_reg); | |
62 | } | |
63 | ||
98fa08a5 PC |
64 | static void lightrec_emit_end_of_block(struct lightrec_cstate *state, |
65 | const struct block *block, u16 offset, | |
d16005f8 PC |
66 | s8 reg_new_pc, u32 imm, u8 ra_reg, |
67 | u32 link, bool update_cycles) | |
68 | { | |
d16005f8 | 69 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 70 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 71 | const struct opcode *op = &block->opcode_list[offset], |
cb72ea13 | 72 | *ds = get_delay_slot(block->opcode_list, offset); |
684432ad | 73 | u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c); |
d75d83f7 | 74 | bool has_ds = has_delay_slot(op->c); |
d16005f8 PC |
75 | |
76 | jit_note(__FILE__, __LINE__); | |
77 | ||
9259d748 PC |
78 | if (link && ra_reg != reg_new_pc) |
79 | update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); | |
d16005f8 | 80 | |
9259d748 PC |
81 | if (reg_new_pc < 0) |
82 | lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm); | |
83 | else | |
84 | lightrec_load_next_pc(reg_cache, _jit, reg_new_pc); | |
d16005f8 | 85 | |
9259d748 PC |
86 | if (link && ra_reg == reg_new_pc) { |
87 | /* Handle the special case: JALR $r0, $r0 | |
88 | * In that case the target PC should be the old value of the | |
89 | * register. */ | |
90 | update_ra_register(reg_cache, _jit, ra_reg, block->pc, link); | |
d16005f8 PC |
91 | } |
92 | ||
d75d83f7 | 93 | if (has_ds && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) { |
684432ad | 94 | cycles += lightrec_cycles_of_opcode(state->state, ds->c); |
d16005f8 PC |
95 | |
96 | /* Recompile the delay slot */ | |
cb72ea13 | 97 | if (ds->c.opcode) |
98fa08a5 | 98 | lightrec_rec_opcode(state, block, offset + 1); |
d16005f8 PC |
99 | } |
100 | ||
03535202 PC |
101 | /* Clean the remaining registers */ |
102 | lightrec_clean_regs(reg_cache, _jit); | |
d16005f8 | 103 | |
d16005f8 PC |
104 | if (cycles && update_cycles) { |
105 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); | |
106 | pr_debug("EOB: %u cycles\n", cycles); | |
107 | } | |
108 | ||
d75d83f7 | 109 | if (has_ds && op_flag_load_delay(ds->flags) |
a5a6f7b8 | 110 | && opcode_has_load_delay(ds->c) && !state->no_load_delay) { |
cb72ea13 PC |
111 | /* If the delay slot is a load opcode, its target register |
112 | * will be written after the first opcode of the target is | |
113 | * executed. Handle this by jumping to a special section of | |
114 | * the dispatcher. It expects the loaded value to be in | |
115 | * REG_TEMP, and the target register number to be in JIT_V1.*/ | |
116 | jit_movi(JIT_V1, ds->c.i.rt); | |
117 | ||
118 | lightrec_jump_to_ds_check(state, _jit); | |
119 | } else { | |
120 | lightrec_jump_to_eob(state, _jit); | |
121 | } | |
0e720fb1 PC |
122 | |
123 | lightrec_regcache_reset(reg_cache); | |
d16005f8 PC |
124 | } |
125 | ||
cb72ea13 PC |
126 | void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state, |
127 | const struct block *block, u16 offset) | |
128 | { | |
129 | struct regcache *reg_cache = state->reg_cache; | |
130 | jit_state_t *_jit = block->_jit; | |
131 | ||
132 | lightrec_clean_regs(reg_cache, _jit); | |
133 | ||
134 | /* Call the interpreter with the block's address in JIT_V1 and the | |
135 | * PC (which might have an offset) in JIT_V0. */ | |
136 | lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, | |
137 | block->pc + (offset << 2)); | |
0e720fb1 | 138 | if (lightrec_store_next_pc()) { |
8afce295 | 139 | jit_stxi_i(lightrec_offset(next_pc), LIGHTREC_REG_STATE, JIT_V0); |
0e720fb1 PC |
140 | } |
141 | ||
cb72ea13 PC |
142 | jit_movi(JIT_V1, (uintptr_t)block); |
143 | ||
144 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); | |
145 | lightrec_jump_to_fn(_jit, state->state->interpreter_func); | |
146 | } | |
147 | ||
148 | static void lightrec_emit_eob(struct lightrec_cstate *state, | |
149 | const struct block *block, u16 offset) | |
d16005f8 | 150 | { |
d16005f8 PC |
151 | struct regcache *reg_cache = state->reg_cache; |
152 | jit_state_t *_jit = block->_jit; | |
153 | ||
03535202 | 154 | lightrec_clean_regs(reg_cache, _jit); |
d16005f8 | 155 | |
9259d748 PC |
156 | lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc, |
157 | block->pc + (offset << 2)); | |
0e720fb1 | 158 | if (lightrec_store_next_pc()) { |
8afce295 | 159 | jit_stxi_i(lightrec_offset(next_pc), LIGHTREC_REG_STATE, JIT_V0); |
0e720fb1 PC |
160 | } |
161 | ||
13b02197 | 162 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); |
d16005f8 | 163 | |
ba3814c1 | 164 | lightrec_jump_to_eob(state, _jit); |
d16005f8 PC |
165 | } |
166 | ||
98fa08a5 | 167 | static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 168 | { |
9259d748 | 169 | union code c = block->opcode_list[offset].c; |
d16005f8 PC |
170 | |
171 | _jit_name(block->_jit, __func__); | |
9259d748 | 172 | lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true); |
98fa08a5 PC |
173 | } |
174 | ||
175 | static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
176 | { | |
98fa08a5 PC |
177 | union code c = block->opcode_list[offset].c; |
178 | ||
179 | _jit_name(block->_jit, __func__); | |
9259d748 | 180 | lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd, |
98fa08a5 | 181 | get_branch_pc(block, offset, 2), true); |
d16005f8 PC |
182 | } |
183 | ||
98fa08a5 | 184 | static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 185 | { |
98fa08a5 PC |
186 | union code c = block->opcode_list[offset].c; |
187 | ||
d16005f8 | 188 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
189 | lightrec_emit_end_of_block(state, block, offset, -1, |
190 | (block->pc & 0xf0000000) | (c.j.imm << 2), | |
191 | 31, 0, true); | |
d16005f8 PC |
192 | } |
193 | ||
98fa08a5 | 194 | static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 195 | { |
98fa08a5 PC |
196 | union code c = block->opcode_list[offset].c; |
197 | ||
d16005f8 | 198 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
199 | lightrec_emit_end_of_block(state, block, offset, -1, |
200 | (block->pc & 0xf0000000) | (c.j.imm << 2), | |
201 | 31, get_branch_pc(block, offset, 2), true); | |
d16005f8 PC |
202 | } |
203 | ||
03535202 PC |
204 | static void lightrec_do_early_unload(struct lightrec_cstate *state, |
205 | const struct block *block, u16 offset) | |
206 | { | |
207 | struct regcache *reg_cache = state->reg_cache; | |
208 | const struct opcode *op = &block->opcode_list[offset]; | |
209 | jit_state_t *_jit = block->_jit; | |
210 | unsigned int i; | |
211 | u8 reg; | |
212 | struct { | |
213 | u8 reg, op; | |
214 | } reg_ops[3] = { | |
215 | { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), }, | |
216 | { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), }, | |
217 | { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), }, | |
218 | }; | |
219 | ||
220 | for (i = 0; i < ARRAY_SIZE(reg_ops); i++) { | |
221 | reg = reg_ops[i].reg; | |
222 | ||
223 | switch (reg_ops[i].op) { | |
224 | case LIGHTREC_REG_UNLOAD: | |
225 | lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true); | |
226 | break; | |
227 | ||
228 | case LIGHTREC_REG_DISCARD: | |
229 | lightrec_discard_reg_if_loaded(reg_cache, reg); | |
230 | break; | |
231 | ||
232 | case LIGHTREC_REG_CLEAN: | |
233 | lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false); | |
234 | break; | |
235 | default: | |
236 | break; | |
237 | }; | |
238 | } | |
239 | } | |
240 | ||
98fa08a5 | 241 | static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset, |
ba3814c1 | 242 | jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz) |
d16005f8 | 243 | { |
98fa08a5 | 244 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 PC |
245 | struct native_register *regs_backup; |
246 | jit_state_t *_jit = block->_jit; | |
247 | struct lightrec_branch *branch; | |
98fa08a5 | 248 | const struct opcode *op = &block->opcode_list[offset], |
cb72ea13 | 249 | *ds = get_delay_slot(block->opcode_list, offset); |
d16005f8 | 250 | jit_node_t *addr; |
cb72ea13 | 251 | bool is_forward = (s16)op->i.imm >= 0; |
684432ad | 252 | int op_cycles = lightrec_cycles_of_opcode(state->state, op->c); |
03535202 | 253 | u32 target_offset, cycles = state->cycles + op_cycles; |
ba3814c1 | 254 | bool no_indirection = false; |
98fa08a5 | 255 | u32 next_pc; |
9259d748 | 256 | u8 rs, rt; |
d16005f8 PC |
257 | |
258 | jit_note(__FILE__, __LINE__); | |
259 | ||
03535202 | 260 | if (!op_flag_no_ds(op->flags)) |
684432ad | 261 | cycles += lightrec_cycles_of_opcode(state->state, ds->c); |
d16005f8 | 262 | |
03535202 PC |
263 | state->cycles = -op_cycles; |
264 | ||
265 | if (!unconditional) { | |
266 | rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT); | |
267 | rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache, | |
268 | _jit, op->i.rt, REG_EXT); | |
269 | ||
270 | /* Unload dead registers before evaluating the branch */ | |
271 | if (OPT_EARLY_UNLOAD) | |
272 | lightrec_do_early_unload(state, block, offset); | |
ba3814c1 PC |
273 | |
274 | if (op_flag_local_branch(op->flags) && | |
cb72ea13 | 275 | (op_flag_no_ds(op->flags) || !ds->opcode) && |
ba3814c1 PC |
276 | is_forward && !lightrec_has_dirty_regs(reg_cache)) |
277 | no_indirection = true; | |
278 | ||
279 | if (no_indirection) | |
280 | pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2); | |
03535202 | 281 | } |
d16005f8 PC |
282 | |
283 | if (cycles) | |
284 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles); | |
285 | ||
286 | if (!unconditional) { | |
d16005f8 | 287 | /* Generate the branch opcode */ |
ba3814c1 PC |
288 | if (!no_indirection) |
289 | addr = jit_new_node_pww(code, NULL, rs, rt); | |
d16005f8 PC |
290 | |
291 | lightrec_free_regs(reg_cache); | |
292 | regs_backup = lightrec_regcache_enter_branch(reg_cache); | |
293 | } | |
294 | ||
03535202 PC |
295 | if (op_flag_local_branch(op->flags)) { |
296 | /* Recompile the delay slot */ | |
cb72ea13 PC |
297 | if (!op_flag_no_ds(op->flags) && ds->opcode) { |
298 | /* Never handle load delays with local branches. */ | |
299 | state->no_load_delay = true; | |
03535202 | 300 | lightrec_rec_opcode(state, block, offset + 1); |
cb72ea13 | 301 | } |
d16005f8 | 302 | |
9259d748 PC |
303 | if (link) |
304 | update_ra_register(reg_cache, _jit, 31, block->pc, link); | |
d16005f8 | 305 | |
03535202 PC |
306 | /* Clean remaining registers */ |
307 | lightrec_clean_regs(reg_cache, _jit); | |
d16005f8 | 308 | |
98fa08a5 | 309 | target_offset = offset + 1 + (s16)op->i.imm |
03535202 | 310 | - !!op_flag_no_ds(op->flags); |
98fa08a5 PC |
311 | pr_debug("Adding local branch to offset 0x%x\n", |
312 | target_offset << 2); | |
313 | branch = &state->local_branches[ | |
314 | state->nb_local_branches++]; | |
d16005f8 | 315 | |
98fa08a5 | 316 | branch->target = target_offset; |
ba3814c1 PC |
317 | |
318 | if (no_indirection) | |
319 | branch->branch = jit_new_node_pww(code2, NULL, rs, rt); | |
320 | else if (is_forward) | |
22eee2ac | 321 | branch->branch = jit_b(); |
d16005f8 PC |
322 | else |
323 | branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0); | |
324 | } | |
325 | ||
03535202 | 326 | if (!op_flag_local_branch(op->flags) || !is_forward) { |
98fa08a5 | 327 | next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm); |
cb72ea13 | 328 | state->no_load_delay = op_flag_local_branch(op->flags); |
98fa08a5 | 329 | lightrec_emit_end_of_block(state, block, offset, -1, next_pc, |
d16005f8 PC |
330 | 31, link, false); |
331 | } | |
332 | ||
333 | if (!unconditional) { | |
ba3814c1 PC |
334 | if (!no_indirection) |
335 | jit_patch(addr); | |
336 | ||
d16005f8 PC |
337 | lightrec_regcache_leave_branch(reg_cache, regs_backup); |
338 | ||
9259d748 PC |
339 | if (bz && link) |
340 | update_ra_register(reg_cache, _jit, 31, block->pc, link); | |
d16005f8 | 341 | |
cb72ea13 PC |
342 | if (!op_flag_no_ds(op->flags) && ds->opcode) { |
343 | state->no_load_delay = true; | |
98fa08a5 | 344 | lightrec_rec_opcode(state, block, offset + 1); |
cb72ea13 | 345 | } |
d16005f8 PC |
346 | } |
347 | } | |
348 | ||
98fa08a5 PC |
349 | static void rec_BNE(struct lightrec_cstate *state, |
350 | const struct block *block, u16 offset) | |
d16005f8 | 351 | { |
98fa08a5 PC |
352 | union code c = block->opcode_list[offset].c; |
353 | ||
d16005f8 | 354 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
355 | |
356 | if (c.i.rt == 0) | |
ba3814c1 | 357 | rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true); |
98fa08a5 | 358 | else |
ba3814c1 | 359 | rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false); |
d16005f8 PC |
360 | } |
361 | ||
98fa08a5 PC |
362 | static void rec_BEQ(struct lightrec_cstate *state, |
363 | const struct block *block, u16 offset) | |
d16005f8 | 364 | { |
98fa08a5 PC |
365 | union code c = block->opcode_list[offset].c; |
366 | ||
d16005f8 | 367 | _jit_name(block->_jit, __func__); |
98fa08a5 PC |
368 | |
369 | if (c.i.rt == 0) | |
ba3814c1 | 370 | rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true); |
98fa08a5 | 371 | else |
ba3814c1 | 372 | rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false); |
d16005f8 PC |
373 | } |
374 | ||
98fa08a5 PC |
375 | static void rec_BLEZ(struct lightrec_cstate *state, |
376 | const struct block *block, u16 offset) | |
d16005f8 | 377 | { |
98fa08a5 PC |
378 | union code c = block->opcode_list[offset].c; |
379 | ||
d16005f8 | 380 | _jit_name(block->_jit, __func__); |
ba3814c1 | 381 | rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true); |
d16005f8 PC |
382 | } |
383 | ||
98fa08a5 PC |
384 | static void rec_BGTZ(struct lightrec_cstate *state, |
385 | const struct block *block, u16 offset) | |
d16005f8 PC |
386 | { |
387 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 388 | rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true); |
d16005f8 PC |
389 | } |
390 | ||
98fa08a5 PC |
391 | static void rec_regimm_BLTZ(struct lightrec_cstate *state, |
392 | const struct block *block, u16 offset) | |
d16005f8 PC |
393 | { |
394 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 395 | rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true); |
d16005f8 PC |
396 | } |
397 | ||
98fa08a5 PC |
398 | static void rec_regimm_BLTZAL(struct lightrec_cstate *state, |
399 | const struct block *block, u16 offset) | |
d16005f8 PC |
400 | { |
401 | _jit_name(block->_jit, __func__); | |
ba3814c1 | 402 | rec_b(state, block, offset, jit_code_bgei, jit_code_blti, |
98fa08a5 | 403 | get_branch_pc(block, offset, 2), false, true); |
d16005f8 PC |
404 | } |
405 | ||
98fa08a5 PC |
406 | static void rec_regimm_BGEZ(struct lightrec_cstate *state, |
407 | const struct block *block, u16 offset) | |
d16005f8 | 408 | { |
98fa08a5 PC |
409 | union code c = block->opcode_list[offset].c; |
410 | ||
d16005f8 | 411 | _jit_name(block->_jit, __func__); |
ba3814c1 | 412 | rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true); |
d16005f8 PC |
413 | } |
414 | ||
98fa08a5 PC |
415 | static void rec_regimm_BGEZAL(struct lightrec_cstate *state, |
416 | const struct block *block, u16 offset) | |
d16005f8 | 417 | { |
98fa08a5 | 418 | const struct opcode *op = &block->opcode_list[offset]; |
d16005f8 | 419 | _jit_name(block->_jit, __func__); |
ba3814c1 | 420 | rec_b(state, block, offset, jit_code_blti, jit_code_bgei, |
98fa08a5 PC |
421 | get_branch_pc(block, offset, 2), |
422 | !op->i.rs, true); | |
d16005f8 PC |
423 | } |
424 | ||
684432ad PC |
425 | static void rec_alloc_rs_rd(struct regcache *reg_cache, |
426 | jit_state_t *_jit, | |
427 | const struct opcode *op, | |
428 | u8 rs, u8 rd, | |
429 | u8 in_flags, u8 out_flags, | |
430 | u8 *rs_out, u8 *rd_out) | |
431 | { | |
432 | bool unload, discard; | |
433 | u32 unload_flags; | |
434 | ||
435 | if (OPT_EARLY_UNLOAD) { | |
436 | unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags); | |
437 | unload = unload_flags == LIGHTREC_REG_UNLOAD; | |
438 | discard = unload_flags == LIGHTREC_REG_DISCARD; | |
439 | } | |
440 | ||
441 | if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) { | |
442 | rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags); | |
443 | lightrec_remap_reg(reg_cache, _jit, rs, rd, discard); | |
444 | lightrec_set_reg_out_flags(reg_cache, rs, out_flags); | |
445 | rd = rs; | |
446 | } else { | |
447 | rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags); | |
448 | rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags); | |
449 | } | |
450 | ||
451 | *rs_out = rs; | |
452 | *rd_out = rd; | |
453 | } | |
454 | ||
98fa08a5 PC |
455 | static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block, |
456 | u16 offset, jit_code_t code, bool slti) | |
d16005f8 | 457 | { |
98fa08a5 PC |
458 | struct regcache *reg_cache = state->reg_cache; |
459 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 460 | jit_state_t *_jit = block->_jit; |
98fa08a5 PC |
461 | u8 rs, rt, out_flags = REG_EXT; |
462 | ||
463 | if (slti) | |
464 | out_flags |= REG_ZEXT; | |
d16005f8 PC |
465 | |
466 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
467 | |
468 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
469 | c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt); | |
d16005f8 | 470 | |
98fa08a5 | 471 | jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm); |
d16005f8 PC |
472 | |
473 | lightrec_free_reg(reg_cache, rs); | |
474 | lightrec_free_reg(reg_cache, rt); | |
475 | } | |
476 | ||
98fa08a5 PC |
477 | static void rec_alu_special(struct lightrec_cstate *state, const struct block *block, |
478 | u16 offset, jit_code_t code, bool out_ext) | |
d16005f8 | 479 | { |
98fa08a5 PC |
480 | struct regcache *reg_cache = state->reg_cache; |
481 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
482 | jit_state_t *_jit = block->_jit; |
483 | u8 rd, rt, rs; | |
484 | ||
485 | jit_note(__FILE__, __LINE__); | |
684432ad | 486 | |
98fa08a5 | 487 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT); |
684432ad PC |
488 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
489 | c.r.rs, c.r.rd, REG_EXT, | |
490 | out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd); | |
d16005f8 PC |
491 | |
492 | jit_new_node_www(code, rd, rs, rt); | |
493 | ||
494 | lightrec_free_reg(reg_cache, rs); | |
495 | lightrec_free_reg(reg_cache, rt); | |
496 | lightrec_free_reg(reg_cache, rd); | |
497 | } | |
498 | ||
98fa08a5 PC |
499 | static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block, |
500 | u16 offset, jit_code_t code) | |
d16005f8 | 501 | { |
98fa08a5 PC |
502 | struct regcache *reg_cache = state->reg_cache; |
503 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 504 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 505 | u8 rd, rt, rs, temp, flags = 0; |
d16005f8 PC |
506 | |
507 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 508 | |
98fa08a5 PC |
509 | if (code == jit_code_rshr) |
510 | flags = REG_EXT; | |
511 | else if (code == jit_code_rshr_u) | |
512 | flags = REG_ZEXT; | |
d16005f8 | 513 | |
684432ad PC |
514 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0); |
515 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
516 | c.r.rt, c.r.rd, flags, flags, &rt, &rd); | |
d16005f8 | 517 | |
684432ad | 518 | if (rt != rd) { |
98fa08a5 PC |
519 | jit_andi(rd, rs, 0x1f); |
520 | jit_new_node_www(code, rd, rt, rd); | |
521 | } else { | |
522 | temp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
523 | jit_andi(temp, rs, 0x1f); | |
d16005f8 | 524 | jit_new_node_www(code, rd, rt, temp); |
98fa08a5 PC |
525 | lightrec_free_reg(reg_cache, temp); |
526 | } | |
d16005f8 PC |
527 | |
528 | lightrec_free_reg(reg_cache, rs); | |
d16005f8 PC |
529 | lightrec_free_reg(reg_cache, rt); |
530 | lightrec_free_reg(reg_cache, rd); | |
531 | } | |
532 | ||
02487de7 PC |
533 | static void rec_movi(struct lightrec_cstate *state, |
534 | const struct block *block, u16 offset) | |
535 | { | |
536 | struct regcache *reg_cache = state->reg_cache; | |
537 | union code c = block->opcode_list[offset].c; | |
538 | jit_state_t *_jit = block->_jit; | |
539 | u16 flags = REG_EXT; | |
684432ad | 540 | s32 value = (s32)(s16) c.i.imm; |
02487de7 PC |
541 | u8 rt; |
542 | ||
684432ad PC |
543 | if (block->opcode_list[offset].flags & LIGHTREC_MOVI) |
544 | value += (s32)((u32)state->movi_temp[c.i.rt] << 16); | |
545 | ||
546 | if (value >= 0) | |
02487de7 PC |
547 | flags |= REG_ZEXT; |
548 | ||
549 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); | |
550 | ||
684432ad | 551 | jit_movi(rt, value); |
02487de7 PC |
552 | |
553 | lightrec_free_reg(reg_cache, rt); | |
554 | } | |
555 | ||
98fa08a5 PC |
556 | static void rec_ADDIU(struct lightrec_cstate *state, |
557 | const struct block *block, u16 offset) | |
d16005f8 | 558 | { |
684432ad PC |
559 | const struct opcode *op = &block->opcode_list[offset]; |
560 | ||
d16005f8 | 561 | _jit_name(block->_jit, __func__); |
02487de7 | 562 | |
684432ad | 563 | if (op->i.rs && !(op->flags & LIGHTREC_MOVI)) |
02487de7 PC |
564 | rec_alu_imm(state, block, offset, jit_code_addi, false); |
565 | else | |
566 | rec_movi(state, block, offset); | |
d16005f8 PC |
567 | } |
568 | ||
98fa08a5 PC |
569 | static void rec_ADDI(struct lightrec_cstate *state, |
570 | const struct block *block, u16 offset) | |
d16005f8 PC |
571 | { |
572 | /* TODO: Handle the exception? */ | |
573 | _jit_name(block->_jit, __func__); | |
02487de7 | 574 | rec_ADDIU(state, block, offset); |
d16005f8 PC |
575 | } |
576 | ||
98fa08a5 PC |
577 | static void rec_SLTIU(struct lightrec_cstate *state, |
578 | const struct block *block, u16 offset) | |
d16005f8 PC |
579 | { |
580 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 581 | rec_alu_imm(state, block, offset, jit_code_lti_u, true); |
d16005f8 PC |
582 | } |
583 | ||
98fa08a5 PC |
584 | static void rec_SLTI(struct lightrec_cstate *state, |
585 | const struct block *block, u16 offset) | |
d16005f8 PC |
586 | { |
587 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 588 | rec_alu_imm(state, block, offset, jit_code_lti, true); |
d16005f8 PC |
589 | } |
590 | ||
98fa08a5 PC |
591 | static void rec_ANDI(struct lightrec_cstate *state, |
592 | const struct block *block, u16 offset) | |
d16005f8 | 593 | { |
98fa08a5 PC |
594 | struct regcache *reg_cache = state->reg_cache; |
595 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
596 | jit_state_t *_jit = block->_jit; |
597 | u8 rs, rt; | |
598 | ||
599 | _jit_name(block->_jit, __func__); | |
600 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
601 | |
602 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
603 | c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt); | |
d16005f8 PC |
604 | |
605 | /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically | |
606 | * casts to uint8_t / uint16_t. */ | |
98fa08a5 | 607 | if (c.i.imm == 0xff) |
d16005f8 | 608 | jit_extr_uc(rt, rs); |
98fa08a5 | 609 | else if (c.i.imm == 0xffff) |
d16005f8 PC |
610 | jit_extr_us(rt, rs); |
611 | else | |
98fa08a5 PC |
612 | jit_andi(rt, rs, (u32)(u16) c.i.imm); |
613 | ||
614 | lightrec_free_reg(reg_cache, rs); | |
615 | lightrec_free_reg(reg_cache, rt); | |
616 | } | |
617 | ||
618 | static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block, | |
619 | u16 offset, jit_code_t code) | |
620 | { | |
621 | struct regcache *reg_cache = state->reg_cache; | |
622 | union code c = block->opcode_list[offset].c; | |
623 | jit_state_t *_jit = block->_jit; | |
624 | u8 rs, rt, flags; | |
625 | ||
626 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
627 | |
628 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
629 | c.i.rs, c.i.rt, 0, 0, &rs, &rt); | |
98fa08a5 PC |
630 | |
631 | flags = lightrec_get_reg_in_flags(reg_cache, rs); | |
632 | lightrec_set_reg_out_flags(reg_cache, rt, flags); | |
633 | ||
634 | jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm); | |
d16005f8 PC |
635 | |
636 | lightrec_free_reg(reg_cache, rs); | |
637 | lightrec_free_reg(reg_cache, rt); | |
638 | } | |
639 | ||
98fa08a5 PC |
640 | |
641 | static void rec_ORI(struct lightrec_cstate *state, | |
642 | const struct block *block, u16 offset) | |
d16005f8 | 643 | { |
684432ad PC |
644 | const struct opcode *op = &block->opcode_list[offset]; |
645 | struct regcache *reg_cache = state->reg_cache; | |
646 | jit_state_t *_jit = block->_jit; | |
647 | s32 val; | |
648 | u8 rt; | |
649 | ||
650 | _jit_name(_jit, __func__); | |
651 | ||
652 | if (op->flags & LIGHTREC_MOVI) { | |
653 | rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT); | |
654 | ||
655 | val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm; | |
656 | jit_movi(rt, val); | |
657 | ||
658 | lightrec_free_reg(reg_cache, rt); | |
659 | } else { | |
660 | rec_alu_or_xor(state, block, offset, jit_code_ori); | |
661 | } | |
d16005f8 PC |
662 | } |
663 | ||
98fa08a5 PC |
664 | static void rec_XORI(struct lightrec_cstate *state, |
665 | const struct block *block, u16 offset) | |
d16005f8 PC |
666 | { |
667 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 668 | rec_alu_or_xor(state, block, offset, jit_code_xori); |
d16005f8 PC |
669 | } |
670 | ||
98fa08a5 PC |
671 | static void rec_LUI(struct lightrec_cstate *state, |
672 | const struct block *block, u16 offset) | |
d16005f8 | 673 | { |
98fa08a5 PC |
674 | struct regcache *reg_cache = state->reg_cache; |
675 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 676 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 677 | u8 rt, flags = REG_EXT; |
d16005f8 | 678 | |
684432ad PC |
679 | if (block->opcode_list[offset].flags & LIGHTREC_MOVI) { |
680 | state->movi_temp[c.i.rt] = c.i.imm; | |
681 | return; | |
682 | } | |
683 | ||
d16005f8 PC |
684 | jit_name(__func__); |
685 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 686 | |
98fa08a5 PC |
687 | if (!(c.i.imm & BIT(15))) |
688 | flags |= REG_ZEXT; | |
689 | ||
690 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags); | |
691 | ||
692 | jit_movi(rt, (s32)(c.i.imm << 16)); | |
d16005f8 PC |
693 | |
694 | lightrec_free_reg(reg_cache, rt); | |
695 | } | |
696 | ||
98fa08a5 PC |
697 | static void rec_special_ADDU(struct lightrec_cstate *state, |
698 | const struct block *block, u16 offset) | |
d16005f8 PC |
699 | { |
700 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 701 | rec_alu_special(state, block, offset, jit_code_addr, false); |
d16005f8 PC |
702 | } |
703 | ||
98fa08a5 PC |
704 | static void rec_special_ADD(struct lightrec_cstate *state, |
705 | const struct block *block, u16 offset) | |
d16005f8 PC |
706 | { |
707 | /* TODO: Handle the exception? */ | |
708 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 709 | rec_alu_special(state, block, offset, jit_code_addr, false); |
d16005f8 PC |
710 | } |
711 | ||
98fa08a5 PC |
712 | static void rec_special_SUBU(struct lightrec_cstate *state, |
713 | const struct block *block, u16 offset) | |
d16005f8 PC |
714 | { |
715 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 716 | rec_alu_special(state, block, offset, jit_code_subr, false); |
d16005f8 PC |
717 | } |
718 | ||
98fa08a5 PC |
719 | static void rec_special_SUB(struct lightrec_cstate *state, |
720 | const struct block *block, u16 offset) | |
d16005f8 PC |
721 | { |
722 | /* TODO: Handle the exception? */ | |
723 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 724 | rec_alu_special(state, block, offset, jit_code_subr, false); |
d16005f8 PC |
725 | } |
726 | ||
98fa08a5 PC |
727 | static void rec_special_AND(struct lightrec_cstate *state, |
728 | const struct block *block, u16 offset) | |
d16005f8 | 729 | { |
98fa08a5 PC |
730 | struct regcache *reg_cache = state->reg_cache; |
731 | union code c = block->opcode_list[offset].c; | |
732 | jit_state_t *_jit = block->_jit; | |
733 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; | |
734 | ||
d16005f8 | 735 | _jit_name(block->_jit, __func__); |
98fa08a5 | 736 | jit_note(__FILE__, __LINE__); |
684432ad | 737 | |
98fa08a5 | 738 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
739 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
740 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
98fa08a5 PC |
741 | |
742 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); | |
743 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
744 | ||
745 | /* Z(rd) = Z(rs) | Z(rt) */ | |
746 | flags_rd = REG_ZEXT & (flags_rs | flags_rt); | |
747 | ||
748 | /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */ | |
749 | if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) || | |
750 | ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) || | |
751 | (REG_EXT & flags_rs & flags_rt)) | |
752 | flags_rd |= REG_EXT; | |
753 | ||
754 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
755 | ||
756 | jit_andr(rd, rs, rt); | |
757 | ||
758 | lightrec_free_reg(reg_cache, rs); | |
759 | lightrec_free_reg(reg_cache, rt); | |
760 | lightrec_free_reg(reg_cache, rd); | |
d16005f8 PC |
761 | } |
762 | ||
98fa08a5 PC |
763 | static void rec_special_or_nor(struct lightrec_cstate *state, |
764 | const struct block *block, u16 offset, bool nor) | |
765 | { | |
766 | struct regcache *reg_cache = state->reg_cache; | |
767 | union code c = block->opcode_list[offset].c; | |
768 | jit_state_t *_jit = block->_jit; | |
769 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0; | |
770 | ||
771 | jit_note(__FILE__, __LINE__); | |
684432ad | 772 | |
98fa08a5 | 773 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
774 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
775 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
98fa08a5 PC |
776 | |
777 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); | |
778 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
779 | ||
780 | /* or: Z(rd) = Z(rs) & Z(rt) | |
781 | * nor: Z(rd) = 0 */ | |
782 | if (!nor) | |
783 | flags_rd = REG_ZEXT & flags_rs & flags_rt; | |
784 | ||
6ce0b00a PC |
785 | /* E(rd) = E(rs) & E(rt) */ |
786 | if (REG_EXT & flags_rs & flags_rt) | |
98fa08a5 PC |
787 | flags_rd |= REG_EXT; |
788 | ||
789 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
790 | ||
791 | jit_orr(rd, rs, rt); | |
792 | ||
793 | if (nor) | |
794 | jit_comr(rd, rd); | |
795 | ||
796 | lightrec_free_reg(reg_cache, rs); | |
797 | lightrec_free_reg(reg_cache, rt); | |
798 | lightrec_free_reg(reg_cache, rd); | |
799 | } | |
800 | ||
801 | static void rec_special_OR(struct lightrec_cstate *state, | |
802 | const struct block *block, u16 offset) | |
d16005f8 PC |
803 | { |
804 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 805 | rec_special_or_nor(state, block, offset, false); |
d16005f8 PC |
806 | } |
807 | ||
98fa08a5 PC |
808 | static void rec_special_NOR(struct lightrec_cstate *state, |
809 | const struct block *block, u16 offset) | |
d16005f8 PC |
810 | { |
811 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 812 | rec_special_or_nor(state, block, offset, true); |
d16005f8 PC |
813 | } |
814 | ||
98fa08a5 PC |
815 | static void rec_special_XOR(struct lightrec_cstate *state, |
816 | const struct block *block, u16 offset) | |
d16005f8 | 817 | { |
98fa08a5 PC |
818 | struct regcache *reg_cache = state->reg_cache; |
819 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 820 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 821 | u8 rd, rt, rs, flags_rs, flags_rt, flags_rd; |
d16005f8 | 822 | |
98fa08a5 PC |
823 | _jit_name(block->_jit, __func__); |
824 | ||
825 | jit_note(__FILE__, __LINE__); | |
684432ad | 826 | |
98fa08a5 | 827 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
684432ad PC |
828 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
829 | c.r.rs, c.r.rd, 0, 0, &rs, &rd); | |
d16005f8 | 830 | |
98fa08a5 PC |
831 | flags_rs = lightrec_get_reg_in_flags(reg_cache, rs); |
832 | flags_rt = lightrec_get_reg_in_flags(reg_cache, rt); | |
d16005f8 | 833 | |
98fa08a5 PC |
834 | /* Z(rd) = Z(rs) & Z(rt) */ |
835 | flags_rd = REG_ZEXT & flags_rs & flags_rt; | |
836 | ||
837 | /* E(rd) = E(rs) & E(rt) */ | |
838 | flags_rd |= REG_EXT & flags_rs & flags_rt; | |
839 | ||
840 | lightrec_set_reg_out_flags(reg_cache, rd, flags_rd); | |
841 | ||
842 | jit_xorr(rd, rs, rt); | |
843 | ||
844 | lightrec_free_reg(reg_cache, rs); | |
845 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
846 | lightrec_free_reg(reg_cache, rd); |
847 | } | |
848 | ||
98fa08a5 PC |
849 | static void rec_special_SLTU(struct lightrec_cstate *state, |
850 | const struct block *block, u16 offset) | |
d16005f8 PC |
851 | { |
852 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 853 | rec_alu_special(state, block, offset, jit_code_ltr_u, true); |
d16005f8 PC |
854 | } |
855 | ||
98fa08a5 PC |
856 | static void rec_special_SLT(struct lightrec_cstate *state, |
857 | const struct block *block, u16 offset) | |
d16005f8 PC |
858 | { |
859 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 860 | rec_alu_special(state, block, offset, jit_code_ltr, true); |
d16005f8 PC |
861 | } |
862 | ||
98fa08a5 PC |
863 | static void rec_special_SLLV(struct lightrec_cstate *state, |
864 | const struct block *block, u16 offset) | |
d16005f8 PC |
865 | { |
866 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 867 | rec_alu_shiftv(state, block, offset, jit_code_lshr); |
d16005f8 PC |
868 | } |
869 | ||
98fa08a5 PC |
870 | static void rec_special_SRLV(struct lightrec_cstate *state, |
871 | const struct block *block, u16 offset) | |
d16005f8 PC |
872 | { |
873 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 874 | rec_alu_shiftv(state, block, offset, jit_code_rshr_u); |
d16005f8 PC |
875 | } |
876 | ||
98fa08a5 PC |
877 | static void rec_special_SRAV(struct lightrec_cstate *state, |
878 | const struct block *block, u16 offset) | |
d16005f8 PC |
879 | { |
880 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 881 | rec_alu_shiftv(state, block, offset, jit_code_rshr); |
d16005f8 PC |
882 | } |
883 | ||
98fa08a5 PC |
884 | static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block, |
885 | u16 offset, jit_code_t code) | |
d16005f8 | 886 | { |
98fa08a5 PC |
887 | struct regcache *reg_cache = state->reg_cache; |
888 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 889 | jit_state_t *_jit = block->_jit; |
684432ad | 890 | u8 rd, rt, flags = 0, out_flags = 0; |
d16005f8 PC |
891 | |
892 | jit_note(__FILE__, __LINE__); | |
893 | ||
98fa08a5 PC |
894 | if (code == jit_code_rshi) |
895 | flags = REG_EXT; | |
896 | else if (code == jit_code_rshi_u) | |
897 | flags = REG_ZEXT; | |
d16005f8 | 898 | |
98fa08a5 PC |
899 | /* Input reg is zero-extended, if we SRL at least by one bit, we know |
900 | * the output reg will be both zero-extended and sign-extended. */ | |
684432ad | 901 | out_flags = flags; |
98fa08a5 | 902 | if (code == jit_code_rshi_u && c.r.imm) |
684432ad PC |
903 | out_flags |= REG_EXT; |
904 | ||
905 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
906 | c.r.rt, c.r.rd, flags, out_flags, &rt, &rd); | |
98fa08a5 PC |
907 | |
908 | jit_new_node_www(code, rd, rt, c.r.imm); | |
d16005f8 PC |
909 | |
910 | lightrec_free_reg(reg_cache, rt); | |
911 | lightrec_free_reg(reg_cache, rd); | |
912 | } | |
913 | ||
98fa08a5 PC |
914 | static void rec_special_SLL(struct lightrec_cstate *state, |
915 | const struct block *block, u16 offset) | |
d16005f8 PC |
916 | { |
917 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 918 | rec_alu_shift(state, block, offset, jit_code_lshi); |
d16005f8 PC |
919 | } |
920 | ||
98fa08a5 PC |
921 | static void rec_special_SRL(struct lightrec_cstate *state, |
922 | const struct block *block, u16 offset) | |
d16005f8 PC |
923 | { |
924 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 925 | rec_alu_shift(state, block, offset, jit_code_rshi_u); |
d16005f8 PC |
926 | } |
927 | ||
98fa08a5 PC |
928 | static void rec_special_SRA(struct lightrec_cstate *state, |
929 | const struct block *block, u16 offset) | |
d16005f8 PC |
930 | { |
931 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 932 | rec_alu_shift(state, block, offset, jit_code_rshi); |
d16005f8 PC |
933 | } |
934 | ||
98fa08a5 PC |
935 | static void rec_alu_mult(struct lightrec_cstate *state, |
936 | const struct block *block, u16 offset, bool is_signed) | |
d16005f8 | 937 | { |
98fa08a5 PC |
938 | struct regcache *reg_cache = state->reg_cache; |
939 | union code c = block->opcode_list[offset].c; | |
03535202 | 940 | u32 flags = block->opcode_list[offset].flags; |
98fa08a5 PC |
941 | u8 reg_lo = get_mult_div_lo(c); |
942 | u8 reg_hi = get_mult_div_hi(c); | |
d16005f8 | 943 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 944 | u8 lo, hi, rs, rt, rflags = 0; |
5459088b PC |
945 | bool no_lo = op_flag_no_lo(flags); |
946 | bool no_hi = op_flag_no_hi(flags); | |
d16005f8 PC |
947 | |
948 | jit_note(__FILE__, __LINE__); | |
949 | ||
98fa08a5 PC |
950 | if (is_signed) |
951 | rflags = REG_EXT; | |
952 | else | |
953 | rflags = REG_ZEXT; | |
954 | ||
955 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); | |
956 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); | |
957 | ||
5459088b | 958 | if (!no_lo) |
98fa08a5 | 959 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); |
98fa08a5 | 960 | |
5459088b | 961 | if (!no_hi) |
98fa08a5 PC |
962 | hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT); |
963 | ||
964 | if (__WORDSIZE == 32) { | |
965 | /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit | |
966 | * operation if the MULT was detected a 32-bit only. */ | |
5459088b | 967 | if (no_lo) { |
98fa08a5 | 968 | if (is_signed) |
5459088b | 969 | jit_hmulr(hi, rs, rt); |
98fa08a5 | 970 | else |
5459088b PC |
971 | jit_hmulr_u(hi, rs, rt); |
972 | } else if (no_hi) { | |
98fa08a5 | 973 | jit_mulr(lo, rs, rt); |
5459088b PC |
974 | } else if (is_signed) { |
975 | jit_qmulr(lo, hi, rs, rt); | |
976 | } else { | |
977 | jit_qmulr_u(lo, hi, rs, rt); | |
98fa08a5 | 978 | } |
d16005f8 | 979 | } else { |
98fa08a5 | 980 | /* On 64-bit systems, do a 64*64->64 bit operation. */ |
5459088b | 981 | if (no_lo) { |
98fa08a5 PC |
982 | jit_mulr(hi, rs, rt); |
983 | jit_rshi(hi, hi, 32); | |
984 | } else { | |
985 | jit_mulr(lo, rs, rt); | |
d16005f8 | 986 | |
98fa08a5 | 987 | /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */ |
5459088b | 988 | if (!no_hi) |
98fa08a5 PC |
989 | jit_rshi(hi, lo, 32); |
990 | } | |
d16005f8 PC |
991 | } |
992 | ||
d16005f8 PC |
993 | lightrec_free_reg(reg_cache, rs); |
994 | lightrec_free_reg(reg_cache, rt); | |
5459088b | 995 | if (!no_lo) |
98fa08a5 | 996 | lightrec_free_reg(reg_cache, lo); |
5459088b | 997 | if (!no_hi) |
d16005f8 PC |
998 | lightrec_free_reg(reg_cache, hi); |
999 | } | |
1000 | ||
98fa08a5 PC |
1001 | static void rec_alu_div(struct lightrec_cstate *state, |
1002 | const struct block *block, u16 offset, bool is_signed) | |
d16005f8 | 1003 | { |
98fa08a5 PC |
1004 | struct regcache *reg_cache = state->reg_cache; |
1005 | union code c = block->opcode_list[offset].c; | |
03535202 PC |
1006 | u32 flags = block->opcode_list[offset].flags; |
1007 | bool no_check = op_flag_no_div_check(flags); | |
98fa08a5 PC |
1008 | u8 reg_lo = get_mult_div_lo(c); |
1009 | u8 reg_hi = get_mult_div_hi(c); | |
d16005f8 PC |
1010 | jit_state_t *_jit = block->_jit; |
1011 | jit_node_t *branch, *to_end; | |
fd58fa32 | 1012 | u8 lo = 0, hi = 0, rs, rt, rflags = 0; |
d16005f8 PC |
1013 | |
1014 | jit_note(__FILE__, __LINE__); | |
d16005f8 | 1015 | |
98fa08a5 PC |
1016 | if (is_signed) |
1017 | rflags = REG_EXT; | |
1018 | else | |
1019 | rflags = REG_ZEXT; | |
1020 | ||
1021 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags); | |
1022 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags); | |
1023 | ||
03535202 | 1024 | if (!op_flag_no_lo(flags)) |
98fa08a5 PC |
1025 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); |
1026 | ||
03535202 | 1027 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1028 | hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0); |
d16005f8 PC |
1029 | |
1030 | /* Jump to special handler if dividing by zero */ | |
98fa08a5 PC |
1031 | if (!no_check) |
1032 | branch = jit_beqi(rt, 0); | |
d16005f8 | 1033 | |
03535202 | 1034 | if (op_flag_no_lo(flags)) { |
98fa08a5 PC |
1035 | if (is_signed) |
1036 | jit_remr(hi, rs, rt); | |
1037 | else | |
1038 | jit_remr_u(hi, rs, rt); | |
03535202 | 1039 | } else if (op_flag_no_hi(flags)) { |
98fa08a5 PC |
1040 | if (is_signed) |
1041 | jit_divr(lo, rs, rt); | |
1042 | else | |
1043 | jit_divr_u(lo, rs, rt); | |
d16005f8 | 1044 | } else { |
98fa08a5 PC |
1045 | if (is_signed) |
1046 | jit_qdivr(lo, hi, rs, rt); | |
1047 | else | |
1048 | jit_qdivr_u(lo, hi, rs, rt); | |
d16005f8 | 1049 | } |
d16005f8 | 1050 | |
98fa08a5 | 1051 | if (!no_check) { |
98fa08a5 | 1052 | /* Jump above the div-by-zero handler */ |
22eee2ac | 1053 | to_end = jit_b(); |
d16005f8 | 1054 | |
98fa08a5 PC |
1055 | jit_patch(branch); |
1056 | ||
03535202 | 1057 | if (!op_flag_no_lo(flags)) { |
98fa08a5 | 1058 | if (is_signed) { |
9259d748 | 1059 | jit_ltr(lo, rs, rt); |
98fa08a5 PC |
1060 | jit_lshi(lo, lo, 1); |
1061 | jit_subi(lo, lo, 1); | |
1062 | } else { | |
9259d748 | 1063 | jit_subi(lo, rt, 1); |
98fa08a5 PC |
1064 | } |
1065 | } | |
d16005f8 | 1066 | |
03535202 | 1067 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1068 | jit_movr(hi, rs); |
d16005f8 | 1069 | |
98fa08a5 PC |
1070 | jit_patch(to_end); |
1071 | } | |
d16005f8 PC |
1072 | |
1073 | lightrec_free_reg(reg_cache, rs); | |
1074 | lightrec_free_reg(reg_cache, rt); | |
98fa08a5 | 1075 | |
03535202 | 1076 | if (!op_flag_no_lo(flags)) |
98fa08a5 PC |
1077 | lightrec_free_reg(reg_cache, lo); |
1078 | ||
03535202 | 1079 | if (!op_flag_no_hi(flags)) |
98fa08a5 | 1080 | lightrec_free_reg(reg_cache, hi); |
d16005f8 PC |
1081 | } |
1082 | ||
98fa08a5 PC |
1083 | static void rec_special_MULT(struct lightrec_cstate *state, |
1084 | const struct block *block, u16 offset) | |
d16005f8 PC |
1085 | { |
1086 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1087 | rec_alu_mult(state, block, offset, true); |
d16005f8 PC |
1088 | } |
1089 | ||
98fa08a5 PC |
1090 | static void rec_special_MULTU(struct lightrec_cstate *state, |
1091 | const struct block *block, u16 offset) | |
d16005f8 PC |
1092 | { |
1093 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1094 | rec_alu_mult(state, block, offset, false); |
d16005f8 PC |
1095 | } |
1096 | ||
98fa08a5 PC |
1097 | static void rec_special_DIV(struct lightrec_cstate *state, |
1098 | const struct block *block, u16 offset) | |
d16005f8 PC |
1099 | { |
1100 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1101 | rec_alu_div(state, block, offset, true); |
d16005f8 PC |
1102 | } |
1103 | ||
98fa08a5 PC |
1104 | static void rec_special_DIVU(struct lightrec_cstate *state, |
1105 | const struct block *block, u16 offset) | |
d16005f8 PC |
1106 | { |
1107 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1108 | rec_alu_div(state, block, offset, false); |
d16005f8 PC |
1109 | } |
1110 | ||
98fa08a5 | 1111 | static void rec_alu_mv_lo_hi(struct lightrec_cstate *state, |
684432ad PC |
1112 | const struct block *block, u16 offset, |
1113 | u8 dst, u8 src) | |
d16005f8 | 1114 | { |
98fa08a5 | 1115 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 PC |
1116 | jit_state_t *_jit = block->_jit; |
1117 | ||
1118 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
1119 | |
1120 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
1121 | src, dst, 0, REG_EXT, &src, &dst); | |
d16005f8 | 1122 | |
d16005f8 | 1123 | jit_extr_i(dst, src); |
d16005f8 PC |
1124 | |
1125 | lightrec_free_reg(reg_cache, src); | |
1126 | lightrec_free_reg(reg_cache, dst); | |
1127 | } | |
1128 | ||
98fa08a5 PC |
1129 | static void rec_special_MFHI(struct lightrec_cstate *state, |
1130 | const struct block *block, u16 offset) | |
d16005f8 | 1131 | { |
98fa08a5 PC |
1132 | union code c = block->opcode_list[offset].c; |
1133 | ||
d16005f8 | 1134 | _jit_name(block->_jit, __func__); |
684432ad | 1135 | rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI); |
d16005f8 PC |
1136 | } |
1137 | ||
98fa08a5 PC |
1138 | static void rec_special_MTHI(struct lightrec_cstate *state, |
1139 | const struct block *block, u16 offset) | |
d16005f8 | 1140 | { |
98fa08a5 PC |
1141 | union code c = block->opcode_list[offset].c; |
1142 | ||
d16005f8 | 1143 | _jit_name(block->_jit, __func__); |
684432ad | 1144 | rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs); |
d16005f8 PC |
1145 | } |
1146 | ||
98fa08a5 PC |
1147 | static void rec_special_MFLO(struct lightrec_cstate *state, |
1148 | const struct block *block, u16 offset) | |
d16005f8 | 1149 | { |
98fa08a5 PC |
1150 | union code c = block->opcode_list[offset].c; |
1151 | ||
d16005f8 | 1152 | _jit_name(block->_jit, __func__); |
684432ad | 1153 | rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO); |
d16005f8 PC |
1154 | } |
1155 | ||
98fa08a5 PC |
1156 | static void rec_special_MTLO(struct lightrec_cstate *state, |
1157 | const struct block *block, u16 offset) | |
d16005f8 | 1158 | { |
98fa08a5 PC |
1159 | union code c = block->opcode_list[offset].c; |
1160 | ||
d16005f8 | 1161 | _jit_name(block->_jit, __func__); |
684432ad | 1162 | rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs); |
d16005f8 PC |
1163 | } |
1164 | ||
ba3814c1 PC |
1165 | static void call_to_c_wrapper(struct lightrec_cstate *state, |
1166 | const struct block *block, u32 arg, | |
1167 | enum c_wrappers wrapper) | |
d16005f8 | 1168 | { |
98fa08a5 | 1169 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 1170 | jit_state_t *_jit = block->_jit; |
ba3814c1 | 1171 | s8 tmp, tmp2; |
d16005f8 | 1172 | |
ba3814c1 PC |
1173 | /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */ |
1174 | tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1); | |
d16005f8 | 1175 | |
8afce295 PC |
1176 | jit_movi(tmp2, (unsigned int)wrapper << (1 + __WORDSIZE / 32)); |
1177 | ||
ba3814c1 | 1178 | tmp = lightrec_get_reg_with_value(reg_cache, |
8afce295 | 1179 | (intptr_t) state->state->c_wrapper); |
ba3814c1 PC |
1180 | if (tmp < 0) { |
1181 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
8afce295 | 1182 | jit_ldxi(tmp, LIGHTREC_REG_STATE, lightrec_offset(c_wrapper)); |
ba3814c1 PC |
1183 | |
1184 | lightrec_temp_set_value(reg_cache, tmp, | |
8afce295 | 1185 | (intptr_t) state->state->c_wrapper); |
22eee2ac PC |
1186 | } |
1187 | ||
ba3814c1 PC |
1188 | lightrec_free_reg(reg_cache, tmp2); |
1189 | ||
1190 | #ifdef __mips__ | |
1191 | /* On MIPS, register t9 is always used as the target register for JALR. | |
1192 | * Therefore if it does not contain the target address we must | |
1193 | * invalidate it. */ | |
1194 | if (tmp != _T9) | |
1195 | lightrec_unload_reg(reg_cache, _jit, _T9); | |
1196 | #endif | |
1197 | ||
1198 | jit_prepare(); | |
1199 | jit_pushargi(arg); | |
1200 | ||
22eee2ac | 1201 | lightrec_regcache_mark_live(reg_cache, _jit); |
98fa08a5 | 1202 | jit_callr(tmp); |
d16005f8 | 1203 | |
98fa08a5 | 1204 | lightrec_free_reg(reg_cache, tmp); |
98fa08a5 PC |
1205 | lightrec_regcache_mark_live(reg_cache, _jit); |
1206 | } | |
1207 | ||
1208 | static void rec_io(struct lightrec_cstate *state, | |
1209 | const struct block *block, u16 offset, | |
1210 | bool load_rt, bool read_rt) | |
1211 | { | |
1212 | struct regcache *reg_cache = state->reg_cache; | |
1213 | jit_state_t *_jit = block->_jit; | |
1214 | union code c = block->opcode_list[offset].c; | |
03535202 | 1215 | u32 flags = block->opcode_list[offset].flags; |
22eee2ac | 1216 | bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags); |
98fa08a5 | 1217 | u32 lut_entry; |
cb72ea13 | 1218 | u8 zero; |
d16005f8 | 1219 | |
98fa08a5 | 1220 | jit_note(__FILE__, __LINE__); |
d16005f8 | 1221 | |
98fa08a5 PC |
1222 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); |
1223 | ||
1224 | if (read_rt && likely(c.i.rt)) | |
1225 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); | |
d16005f8 | 1226 | else if (load_rt) |
98fa08a5 | 1227 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); |
d16005f8 | 1228 | |
cb72ea13 PC |
1229 | if (op_flag_load_delay(flags) && !state->no_load_delay) { |
1230 | /* Clear state->in_delay_slot_n. This notifies the lightrec_rw | |
1231 | * wrapper that it should write the REG_TEMP register instead of | |
1232 | * the actual output register of the opcode. */ | |
1233 | zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); | |
8afce295 PC |
1234 | jit_stxi_c(lightrec_offset(in_delay_slot_n), |
1235 | LIGHTREC_REG_STATE, zero); | |
cb72ea13 PC |
1236 | lightrec_free_reg(reg_cache, zero); |
1237 | } | |
1238 | ||
d16005f8 | 1239 | if (is_tagged) { |
ba3814c1 | 1240 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW); |
d16005f8 | 1241 | } else { |
98fa08a5 PC |
1242 | lut_entry = lightrec_get_lut_entry(block); |
1243 | call_to_c_wrapper(state, block, (lut_entry << 16) | offset, | |
ba3814c1 | 1244 | C_WRAPPER_RW_GENERIC); |
d16005f8 | 1245 | } |
d16005f8 PC |
1246 | } |
1247 | ||
f5ee77ca | 1248 | static u32 rec_ram_mask(const struct lightrec_state *state) |
02487de7 PC |
1249 | { |
1250 | return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1; | |
1251 | } | |
1252 | ||
13b02197 PC |
1253 | static u32 rec_io_mask(const struct lightrec_state *state) |
1254 | { | |
1255 | u32 length = state->maps[PSX_MAP_HW_REGISTERS].length; | |
1256 | ||
2ec79b77 | 1257 | return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0); |
13b02197 PC |
1258 | } |
1259 | ||
878e6cda PC |
1260 | static void rec_add_offset(struct lightrec_cstate *cstate, |
1261 | jit_state_t *_jit, u8 reg_out, u8 reg_in, | |
1262 | uintptr_t offset) | |
1263 | { | |
1264 | struct regcache *reg_cache = cstate->reg_cache; | |
1265 | u8 reg_imm; | |
1266 | ||
1267 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, offset); | |
1268 | jit_addr(reg_out, reg_in, reg_imm); | |
1269 | ||
1270 | lightrec_free_reg(reg_cache, reg_imm); | |
1271 | } | |
1272 | ||
1273 | static void rec_and_mask(struct lightrec_cstate *cstate, | |
1274 | jit_state_t *_jit, u8 reg_out, u8 reg_in, u32 mask) | |
1275 | { | |
1276 | struct regcache *reg_cache = cstate->reg_cache; | |
1277 | u8 reg_imm; | |
1278 | ||
1279 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask); | |
1280 | jit_andr(reg_out, reg_in, reg_imm); | |
1281 | ||
1282 | lightrec_free_reg(reg_cache, reg_imm); | |
1283 | } | |
1284 | ||
22eee2ac PC |
1285 | static void rec_store_memory(struct lightrec_cstate *cstate, |
1286 | const struct block *block, | |
1287 | u16 offset, jit_code_t code, | |
878e6cda PC |
1288 | jit_code_t swap_code, uintptr_t addr_offset, |
1289 | u32 addr_mask, bool invalidate) | |
22eee2ac | 1290 | { |
02487de7 | 1291 | const struct lightrec_state *state = cstate->state; |
22eee2ac PC |
1292 | struct regcache *reg_cache = cstate->reg_cache; |
1293 | struct opcode *op = &block->opcode_list[offset]; | |
1294 | jit_state_t *_jit = block->_jit; | |
1295 | union code c = op->c; | |
f5ee77ca | 1296 | u8 rs, rt, tmp = 0, tmp2 = 0, tmp3, addr_reg, addr_reg2; |
22eee2ac | 1297 | s16 imm = (s16)c.i.imm; |
02487de7 | 1298 | s32 simm = (s32)imm << (1 - lut_is_32bit(state)); |
8afce295 | 1299 | s32 lut_offt = lightrec_offset(code_lut); |
03535202 | 1300 | bool no_mask = op_flag_no_mask(op->flags); |
02487de7 | 1301 | bool add_imm = c.i.imm && |
5459088b PC |
1302 | (c.i.op == OP_META_SWU |
1303 | || (!state->mirrors_mapped && !no_mask) || (invalidate && | |
02487de7 | 1304 | ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt)))); |
684432ad | 1305 | bool need_tmp = !no_mask || add_imm || invalidate; |
9259d748 | 1306 | bool swc2 = c.i.op == OP_SWC2; |
cb72ea13 | 1307 | u8 in_reg = swc2 ? REG_TEMP : c.i.rt; |
22eee2ac | 1308 | |
22eee2ac PC |
1309 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
1310 | if (need_tmp) | |
1311 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
1312 | ||
1313 | addr_reg = rs; | |
1314 | ||
1315 | if (add_imm) { | |
1316 | jit_addi(tmp, addr_reg, (s16)c.i.imm); | |
0e720fb1 | 1317 | lightrec_free_reg(reg_cache, rs); |
22eee2ac PC |
1318 | addr_reg = tmp; |
1319 | imm = 0; | |
1320 | } else if (simm) { | |
1321 | lut_offt += simm; | |
1322 | } | |
1323 | ||
1324 | if (!no_mask) { | |
878e6cda | 1325 | rec_and_mask(cstate, _jit, tmp, addr_reg, addr_mask); |
22eee2ac PC |
1326 | addr_reg = tmp; |
1327 | } | |
1328 | ||
22eee2ac | 1329 | if (addr_offset) { |
ba3814c1 | 1330 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
878e6cda | 1331 | rec_add_offset(cstate, _jit, tmp2, addr_reg, addr_offset); |
22eee2ac PC |
1332 | addr_reg2 = tmp2; |
1333 | } else { | |
1334 | addr_reg2 = addr_reg; | |
1335 | } | |
1336 | ||
684432ad PC |
1337 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
1338 | ||
9259d748 | 1339 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1340 | tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1341 | ||
1342 | jit_new_node_ww(swap_code, tmp3, rt); | |
5459088b PC |
1343 | |
1344 | if (c.i.op == OP_META_SWU) | |
1345 | jit_unstr(addr_reg2, tmp3, LIGHTNING_UNALIGNED_32BIT); | |
1346 | else | |
1347 | jit_new_node_www(code, imm, addr_reg2, tmp3); | |
02487de7 PC |
1348 | |
1349 | lightrec_free_reg(reg_cache, tmp3); | |
5459088b PC |
1350 | } else if (c.i.op == OP_META_SWU) { |
1351 | jit_unstr(addr_reg2, rt, LIGHTNING_UNALIGNED_32BIT); | |
02487de7 PC |
1352 | } else { |
1353 | jit_new_node_www(code, imm, addr_reg2, rt); | |
1354 | } | |
1355 | ||
22eee2ac PC |
1356 | lightrec_free_reg(reg_cache, rt); |
1357 | ||
1358 | if (invalidate) { | |
1359 | tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); | |
1360 | ||
1361 | if (c.i.op != OP_SW) { | |
ba3814c1 PC |
1362 | jit_andi(tmp, addr_reg, ~3); |
1363 | addr_reg = tmp; | |
22eee2ac PC |
1364 | } |
1365 | ||
02487de7 | 1366 | if (!lut_is_32bit(state)) { |
ba3814c1 PC |
1367 | jit_lshi(tmp, addr_reg, 1); |
1368 | addr_reg = tmp; | |
22eee2ac PC |
1369 | } |
1370 | ||
02487de7 PC |
1371 | if (addr_reg == rs && c.i.rs == 0) { |
1372 | addr_reg = LIGHTREC_REG_STATE; | |
1373 | } else { | |
cb72ea13 | 1374 | jit_add_state(tmp, addr_reg); |
ba3814c1 | 1375 | addr_reg = tmp; |
22eee2ac PC |
1376 | } |
1377 | ||
02487de7 PC |
1378 | if (lut_is_32bit(state)) |
1379 | jit_stxi_i(lut_offt, addr_reg, tmp3); | |
1380 | else | |
1381 | jit_stxi(lut_offt, addr_reg, tmp3); | |
22eee2ac PC |
1382 | |
1383 | lightrec_free_reg(reg_cache, tmp3); | |
1384 | } | |
1385 | ||
ba3814c1 | 1386 | if (addr_offset) |
22eee2ac PC |
1387 | lightrec_free_reg(reg_cache, tmp2); |
1388 | if (need_tmp) | |
1389 | lightrec_free_reg(reg_cache, tmp); | |
1390 | lightrec_free_reg(reg_cache, rs); | |
1391 | } | |
1392 | ||
1393 | static void rec_store_ram(struct lightrec_cstate *cstate, | |
1394 | const struct block *block, | |
1395 | u16 offset, jit_code_t code, | |
02487de7 | 1396 | jit_code_t swap_code, bool invalidate) |
22eee2ac | 1397 | { |
f5ee77ca | 1398 | const struct lightrec_state *state = cstate->state; |
02487de7 | 1399 | |
22eee2ac PC |
1400 | _jit_note(block->_jit, __FILE__, __LINE__); |
1401 | ||
02487de7 PC |
1402 | return rec_store_memory(cstate, block, offset, code, swap_code, |
1403 | state->offset_ram, rec_ram_mask(state), | |
1404 | invalidate); | |
22eee2ac PC |
1405 | } |
1406 | ||
1407 | static void rec_store_scratch(struct lightrec_cstate *cstate, | |
02487de7 PC |
1408 | const struct block *block, u16 offset, |
1409 | jit_code_t code, jit_code_t swap_code) | |
22eee2ac PC |
1410 | { |
1411 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1412 | ||
02487de7 | 1413 | return rec_store_memory(cstate, block, offset, code, swap_code, |
22eee2ac PC |
1414 | cstate->state->offset_scratch, |
1415 | 0x1fffffff, false); | |
1416 | } | |
1417 | ||
ba3814c1 PC |
1418 | static void rec_store_io(struct lightrec_cstate *cstate, |
1419 | const struct block *block, u16 offset, | |
1420 | jit_code_t code, jit_code_t swap_code) | |
1421 | { | |
1422 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1423 | ||
1424 | return rec_store_memory(cstate, block, offset, code, swap_code, | |
1425 | cstate->state->offset_io, | |
13b02197 | 1426 | rec_io_mask(cstate->state), false); |
ba3814c1 PC |
1427 | } |
1428 | ||
98fa08a5 PC |
1429 | static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate, |
1430 | const struct block *block, | |
02487de7 PC |
1431 | u16 offset, jit_code_t code, |
1432 | jit_code_t swap_code) | |
d16005f8 | 1433 | { |
f5ee77ca | 1434 | const struct lightrec_state *state = cstate->state; |
878e6cda | 1435 | u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE; |
98fa08a5 PC |
1436 | struct regcache *reg_cache = cstate->reg_cache; |
1437 | union code c = block->opcode_list[offset].c; | |
d16005f8 PC |
1438 | jit_state_t *_jit = block->_jit; |
1439 | jit_node_t *to_not_ram, *to_end; | |
9259d748 | 1440 | bool swc2 = c.i.op == OP_SWC2; |
878e6cda | 1441 | u8 addr_reg, tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt; |
d16005f8 PC |
1442 | s16 imm; |
1443 | ||
1444 | jit_note(__FILE__, __LINE__); | |
98fa08a5 | 1445 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
d16005f8 | 1446 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
98fa08a5 | 1447 | |
d16005f8 | 1448 | /* Convert to KUNSEG and avoid RAM mirrors */ |
5459088b | 1449 | if ((c.i.op == OP_META_SWU || !state->mirrors_mapped) && c.i.imm) { |
d16005f8 | 1450 | imm = 0; |
98fa08a5 | 1451 | jit_addi(tmp, rs, (s16)c.i.imm); |
878e6cda | 1452 | addr_reg = tmp; |
d16005f8 | 1453 | } else { |
684432ad | 1454 | imm = (s16)c.i.imm; |
878e6cda | 1455 | addr_reg = rs; |
d16005f8 PC |
1456 | } |
1457 | ||
878e6cda PC |
1458 | rec_and_mask(cstate, _jit, tmp, addr_reg, 0x1f800000 | (ram_size - 1)); |
1459 | ||
d16005f8 PC |
1460 | lightrec_free_reg(reg_cache, rs); |
1461 | ||
1462 | if (state->offset_ram != state->offset_scratch) { | |
684432ad PC |
1463 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1464 | ||
d16005f8 PC |
1465 | to_not_ram = jit_bmsi(tmp, BIT(28)); |
1466 | ||
1467 | jit_movi(tmp2, state->offset_ram); | |
1468 | ||
22eee2ac | 1469 | to_end = jit_b(); |
d16005f8 PC |
1470 | jit_patch(to_not_ram); |
1471 | ||
1472 | jit_movi(tmp2, state->offset_scratch); | |
1473 | jit_patch(to_end); | |
1474 | } else if (state->offset_ram) { | |
684432ad PC |
1475 | tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, |
1476 | state->offset_ram); | |
d16005f8 PC |
1477 | } |
1478 | ||
684432ad | 1479 | if (state->offset_ram || state->offset_scratch) { |
d16005f8 | 1480 | jit_addr(tmp, tmp, tmp2); |
98fa08a5 PC |
1481 | lightrec_free_reg(reg_cache, tmp2); |
1482 | } | |
d16005f8 | 1483 | |
9259d748 | 1484 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
ba3814c1 | 1485 | |
9259d748 | 1486 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1487 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
1488 | ||
1489 | jit_new_node_ww(swap_code, tmp2, rt); | |
5459088b PC |
1490 | |
1491 | if (c.i.op == OP_META_SWU) | |
1492 | jit_unstr(tmp, tmp2, LIGHTNING_UNALIGNED_32BIT); | |
1493 | else | |
1494 | jit_new_node_www(code, imm, tmp, tmp2); | |
02487de7 PC |
1495 | |
1496 | lightrec_free_reg(reg_cache, tmp2); | |
5459088b PC |
1497 | } else if (c.i.op == OP_META_SWU) { |
1498 | jit_unstr(tmp, rt, LIGHTNING_UNALIGNED_32BIT); | |
02487de7 PC |
1499 | } else { |
1500 | jit_new_node_www(code, imm, tmp, rt); | |
1501 | } | |
d16005f8 PC |
1502 | |
1503 | lightrec_free_reg(reg_cache, rt); | |
1504 | lightrec_free_reg(reg_cache, tmp); | |
1505 | } | |
1506 | ||
98fa08a5 | 1507 | static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block, |
02487de7 | 1508 | u16 offset, jit_code_t code, jit_code_t swap_code) |
d16005f8 | 1509 | { |
f5ee77ca | 1510 | const struct lightrec_state *state = cstate->state; |
98fa08a5 PC |
1511 | u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE; |
1512 | struct regcache *reg_cache = cstate->reg_cache; | |
1513 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 1514 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 1515 | jit_node_t *to_not_ram, *to_end; |
9259d748 | 1516 | bool swc2 = c.i.op == OP_SWC2; |
878e6cda | 1517 | u8 addr_reg, tmp, tmp2, tmp3, rs, rt, reg_imm; |
cb72ea13 | 1518 | u8 in_reg = swc2 ? REG_TEMP : c.i.rt; |
878e6cda | 1519 | u32 mask; |
f5ee77ca | 1520 | bool different_offsets = state->offset_ram != state->offset_scratch; |
d16005f8 PC |
1521 | |
1522 | jit_note(__FILE__, __LINE__); | |
1523 | ||
98fa08a5 | 1524 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
d16005f8 | 1525 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); |
98fa08a5 | 1526 | tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0); |
d16005f8 PC |
1527 | |
1528 | /* Convert to KUNSEG and avoid RAM mirrors */ | |
98fa08a5 PC |
1529 | if (c.i.imm) { |
1530 | jit_addi(tmp2, rs, (s16)c.i.imm); | |
878e6cda | 1531 | addr_reg = tmp2; |
d16005f8 | 1532 | } else { |
878e6cda | 1533 | addr_reg = rs; |
d16005f8 PC |
1534 | } |
1535 | ||
878e6cda PC |
1536 | rec_and_mask(cstate, _jit, tmp2, addr_reg, 0x1f800000 | (ram_size - 1)); |
1537 | ||
d16005f8 PC |
1538 | lightrec_free_reg(reg_cache, rs); |
1539 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
1540 | ||
878e6cda PC |
1541 | mask = c.i.op == OP_SW ? RAM_SIZE - 1 : (RAM_SIZE - 1) & ~3; |
1542 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask); | |
1543 | ||
f5ee77ca | 1544 | if (different_offsets) { |
9259d748 | 1545 | to_not_ram = jit_bgti(tmp2, ram_size); |
878e6cda | 1546 | addr_reg = tmp2; |
9259d748 PC |
1547 | } else { |
1548 | jit_lti_u(tmp, tmp2, ram_size); | |
1549 | jit_movnr(tmp, tmp2, tmp); | |
878e6cda | 1550 | addr_reg = tmp; |
9259d748 | 1551 | } |
98fa08a5 | 1552 | |
d16005f8 | 1553 | /* Compute the offset to the code LUT */ |
878e6cda | 1554 | jit_andr(tmp, addr_reg, reg_imm); |
9259d748 | 1555 | |
02487de7 | 1556 | if (!lut_is_32bit(state)) |
98fa08a5 | 1557 | jit_lshi(tmp, tmp, 1); |
cb72ea13 | 1558 | jit_add_state(tmp, tmp); |
d16005f8 PC |
1559 | |
1560 | /* Write NULL to the code LUT to invalidate any block that's there */ | |
02487de7 | 1561 | if (lut_is_32bit(state)) |
8afce295 | 1562 | jit_stxi_i(lightrec_offset(code_lut), tmp, tmp3); |
02487de7 | 1563 | else |
8afce295 | 1564 | jit_stxi(lightrec_offset(code_lut), tmp, tmp3); |
d16005f8 | 1565 | |
5459088b PC |
1566 | if (c.i.op == OP_META_SWU) { |
1567 | /* With a SWU opcode, we might have touched the following 32-bit | |
1568 | * word, so invalidate it as well */ | |
1569 | if (lut_is_32bit(state)) { | |
8afce295 | 1570 | jit_stxi_i(lightrec_offset(code_lut) + 4, tmp, tmp3); |
5459088b | 1571 | } else { |
8afce295 PC |
1572 | jit_stxi(lightrec_offset(code_lut) + sizeof(uintptr_t), |
1573 | tmp, tmp3); | |
5459088b PC |
1574 | } |
1575 | } | |
1576 | ||
f5ee77ca | 1577 | if (different_offsets) { |
d16005f8 PC |
1578 | jit_movi(tmp, state->offset_ram); |
1579 | ||
22eee2ac | 1580 | to_end = jit_b(); |
9259d748 | 1581 | jit_patch(to_not_ram); |
d16005f8 PC |
1582 | } |
1583 | ||
d16005f8 PC |
1584 | if (state->offset_ram || state->offset_scratch) |
1585 | jit_movi(tmp, state->offset_scratch); | |
1586 | ||
f5ee77ca | 1587 | if (different_offsets) |
d16005f8 PC |
1588 | jit_patch(to_end); |
1589 | ||
1590 | if (state->offset_ram || state->offset_scratch) | |
1591 | jit_addr(tmp2, tmp2, tmp); | |
1592 | ||
1593 | lightrec_free_reg(reg_cache, tmp); | |
1594 | lightrec_free_reg(reg_cache, tmp3); | |
878e6cda | 1595 | lightrec_free_reg(reg_cache, reg_imm); |
d16005f8 | 1596 | |
9259d748 | 1597 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0); |
02487de7 | 1598 | |
9259d748 | 1599 | if (is_big_endian() && swap_code && in_reg) { |
02487de7 PC |
1600 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
1601 | ||
1602 | jit_new_node_ww(swap_code, tmp, rt); | |
5459088b PC |
1603 | |
1604 | if (c.i.op == OP_META_SWU) | |
1605 | jit_unstr(tmp2, tmp, LIGHTNING_UNALIGNED_32BIT); | |
1606 | else | |
1607 | jit_new_node_www(code, 0, tmp2, tmp); | |
02487de7 PC |
1608 | |
1609 | lightrec_free_reg(reg_cache, tmp); | |
5459088b PC |
1610 | } else if (c.i.op == OP_META_SWU) { |
1611 | jit_unstr(tmp2, rt, LIGHTNING_UNALIGNED_32BIT); | |
02487de7 PC |
1612 | } else { |
1613 | jit_new_node_www(code, 0, tmp2, rt); | |
1614 | } | |
d16005f8 PC |
1615 | |
1616 | lightrec_free_reg(reg_cache, rt); | |
1617 | lightrec_free_reg(reg_cache, tmp2); | |
1618 | } | |
1619 | ||
98fa08a5 | 1620 | static void rec_store(struct lightrec_cstate *state, |
02487de7 PC |
1621 | const struct block *block, u16 offset, |
1622 | jit_code_t code, jit_code_t swap_code) | |
d16005f8 | 1623 | { |
03535202 | 1624 | u32 flags = block->opcode_list[offset].flags; |
9259d748 | 1625 | u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags); |
03535202 | 1626 | bool no_invalidate = op_flag_no_invalidate(flags) || |
684432ad | 1627 | (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY); |
9259d748 PC |
1628 | union code c = block->opcode_list[offset].c; |
1629 | bool is_swc2 = c.i.op == OP_SWC2; | |
1630 | ||
1631 | if (is_swc2) { | |
1632 | switch (mode) { | |
1633 | case LIGHTREC_IO_RAM: | |
1634 | case LIGHTREC_IO_SCRATCH: | |
1635 | case LIGHTREC_IO_DIRECT: | |
1636 | case LIGHTREC_IO_DIRECT_HW: | |
cb72ea13 | 1637 | rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP); |
9259d748 PC |
1638 | break; |
1639 | default: | |
1640 | break; | |
1641 | } | |
1642 | } | |
22eee2ac | 1643 | |
9259d748 | 1644 | switch (mode) { |
22eee2ac | 1645 | case LIGHTREC_IO_RAM: |
02487de7 PC |
1646 | rec_store_ram(state, block, offset, code, |
1647 | swap_code, !no_invalidate); | |
22eee2ac PC |
1648 | break; |
1649 | case LIGHTREC_IO_SCRATCH: | |
02487de7 | 1650 | rec_store_scratch(state, block, offset, code, swap_code); |
22eee2ac PC |
1651 | break; |
1652 | case LIGHTREC_IO_DIRECT: | |
02487de7 PC |
1653 | if (no_invalidate) { |
1654 | rec_store_direct_no_invalidate(state, block, offset, | |
1655 | code, swap_code); | |
1656 | } else { | |
1657 | rec_store_direct(state, block, offset, code, swap_code); | |
1658 | } | |
22eee2ac | 1659 | break; |
ba3814c1 PC |
1660 | case LIGHTREC_IO_DIRECT_HW: |
1661 | rec_store_io(state, block, offset, code, swap_code); | |
1662 | break; | |
22eee2ac | 1663 | default: |
98fa08a5 | 1664 | rec_io(state, block, offset, true, false); |
9259d748 | 1665 | return; |
d16005f8 | 1666 | } |
9259d748 PC |
1667 | |
1668 | if (is_swc2) | |
cb72ea13 | 1669 | lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); |
d16005f8 PC |
1670 | } |
1671 | ||
98fa08a5 PC |
1672 | static void rec_SB(struct lightrec_cstate *state, |
1673 | const struct block *block, u16 offset) | |
d16005f8 PC |
1674 | { |
1675 | _jit_name(block->_jit, __func__); | |
02487de7 | 1676 | rec_store(state, block, offset, jit_code_stxi_c, 0); |
d16005f8 PC |
1677 | } |
1678 | ||
98fa08a5 PC |
1679 | static void rec_SH(struct lightrec_cstate *state, |
1680 | const struct block *block, u16 offset) | |
d16005f8 PC |
1681 | { |
1682 | _jit_name(block->_jit, __func__); | |
02487de7 PC |
1683 | rec_store(state, block, offset, |
1684 | jit_code_stxi_s, jit_code_bswapr_us); | |
d16005f8 PC |
1685 | } |
1686 | ||
98fa08a5 PC |
1687 | static void rec_SW(struct lightrec_cstate *state, |
1688 | const struct block *block, u16 offset) | |
1689 | ||
d16005f8 | 1690 | { |
9259d748 PC |
1691 | union code c = block->opcode_list[offset].c; |
1692 | ||
1693 | _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW"); | |
02487de7 PC |
1694 | rec_store(state, block, offset, |
1695 | jit_code_stxi_i, jit_code_bswapr_ui); | |
d16005f8 PC |
1696 | } |
1697 | ||
98fa08a5 PC |
1698 | static void rec_SWL(struct lightrec_cstate *state, |
1699 | const struct block *block, u16 offset) | |
d16005f8 PC |
1700 | { |
1701 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1702 | rec_io(state, block, offset, true, false); |
d16005f8 PC |
1703 | } |
1704 | ||
98fa08a5 PC |
1705 | static void rec_SWR(struct lightrec_cstate *state, |
1706 | const struct block *block, u16 offset) | |
d16005f8 PC |
1707 | { |
1708 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 1709 | rec_io(state, block, offset, true, false); |
d16005f8 PC |
1710 | } |
1711 | ||
22eee2ac | 1712 | static void rec_load_memory(struct lightrec_cstate *cstate, |
02487de7 PC |
1713 | const struct block *block, u16 offset, |
1714 | jit_code_t code, jit_code_t swap_code, bool is_unsigned, | |
22eee2ac PC |
1715 | uintptr_t addr_offset, u32 addr_mask) |
1716 | { | |
878e6cda | 1717 | struct lightrec_state *state = cstate->state; |
22eee2ac PC |
1718 | struct regcache *reg_cache = cstate->reg_cache; |
1719 | struct opcode *op = &block->opcode_list[offset]; | |
cb72ea13 | 1720 | bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; |
22eee2ac | 1721 | jit_state_t *_jit = block->_jit; |
9259d748 | 1722 | u8 rs, rt, out_reg, addr_reg, flags = REG_EXT; |
03535202 | 1723 | bool no_mask = op_flag_no_mask(op->flags); |
22eee2ac | 1724 | union code c = op->c; |
02487de7 | 1725 | s16 imm; |
22eee2ac | 1726 | |
cb72ea13 PC |
1727 | if (load_delay || c.i.op == OP_LWC2) |
1728 | out_reg = REG_TEMP; | |
9259d748 PC |
1729 | else if (c.i.rt) |
1730 | out_reg = c.i.rt; | |
1731 | else | |
22eee2ac PC |
1732 | return; |
1733 | ||
1734 | if (is_unsigned) | |
1735 | flags |= REG_ZEXT; | |
1736 | ||
1737 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); | |
9259d748 | 1738 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
22eee2ac | 1739 | |
5459088b | 1740 | if ((op->i.op == OP_META_LWU && c.i.imm) |
878e6cda | 1741 | || (!state->mirrors_mapped && c.i.imm && !no_mask)) { |
02487de7 | 1742 | jit_addi(rt, rs, (s16)c.i.imm); |
22eee2ac | 1743 | addr_reg = rt; |
02487de7 | 1744 | imm = 0; |
22eee2ac PC |
1745 | } else { |
1746 | addr_reg = rs; | |
02487de7 PC |
1747 | imm = (s16)c.i.imm; |
1748 | } | |
1749 | ||
5459088b PC |
1750 | if (op->i.op == OP_META_LWU) |
1751 | imm = LIGHTNING_UNALIGNED_32BIT; | |
1752 | ||
02487de7 | 1753 | if (!no_mask) { |
878e6cda | 1754 | rec_and_mask(cstate, _jit, rt, addr_reg, addr_mask); |
02487de7 | 1755 | addr_reg = rt; |
22eee2ac PC |
1756 | } |
1757 | ||
1758 | if (addr_offset) { | |
878e6cda | 1759 | rec_add_offset(cstate, _jit, rt, addr_reg, addr_offset); |
22eee2ac PC |
1760 | addr_reg = rt; |
1761 | } | |
1762 | ||
02487de7 PC |
1763 | jit_new_node_www(code, rt, addr_reg, imm); |
1764 | ||
1765 | if (is_big_endian() && swap_code) { | |
1766 | jit_new_node_ww(swap_code, rt, rt); | |
1767 | ||
1768 | if (c.i.op == OP_LH) | |
1769 | jit_extr_s(rt, rt); | |
1770 | else if (c.i.op == OP_LW && __WORDSIZE == 64) | |
1771 | jit_extr_i(rt, rt); | |
1772 | } | |
22eee2ac PC |
1773 | |
1774 | lightrec_free_reg(reg_cache, rs); | |
1775 | lightrec_free_reg(reg_cache, rt); | |
1776 | } | |
1777 | ||
1778 | static void rec_load_ram(struct lightrec_cstate *cstate, | |
02487de7 PC |
1779 | const struct block *block, u16 offset, |
1780 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1781 | { |
1782 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1783 | ||
02487de7 PC |
1784 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
1785 | cstate->state->offset_ram, rec_ram_mask(cstate->state)); | |
22eee2ac PC |
1786 | } |
1787 | ||
1788 | static void rec_load_bios(struct lightrec_cstate *cstate, | |
02487de7 PC |
1789 | const struct block *block, u16 offset, |
1790 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1791 | { |
1792 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1793 | ||
02487de7 | 1794 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
22eee2ac PC |
1795 | cstate->state->offset_bios, 0x1fffffff); |
1796 | } | |
1797 | ||
1798 | static void rec_load_scratch(struct lightrec_cstate *cstate, | |
02487de7 PC |
1799 | const struct block *block, u16 offset, |
1800 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
22eee2ac PC |
1801 | { |
1802 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1803 | ||
02487de7 | 1804 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, |
22eee2ac PC |
1805 | cstate->state->offset_scratch, 0x1fffffff); |
1806 | } | |
1807 | ||
ba3814c1 PC |
1808 | static void rec_load_io(struct lightrec_cstate *cstate, |
1809 | const struct block *block, u16 offset, | |
1810 | jit_code_t code, jit_code_t swap_code, bool is_unsigned) | |
1811 | { | |
1812 | _jit_note(block->_jit, __FILE__, __LINE__); | |
1813 | ||
1814 | rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned, | |
13b02197 | 1815 | cstate->state->offset_io, rec_io_mask(cstate->state)); |
ba3814c1 PC |
1816 | } |
1817 | ||
02487de7 PC |
1818 | static void rec_load_direct(struct lightrec_cstate *cstate, |
1819 | const struct block *block, u16 offset, | |
1820 | jit_code_t code, jit_code_t swap_code, | |
1821 | bool is_unsigned) | |
d16005f8 | 1822 | { |
f5ee77ca | 1823 | const struct lightrec_state *state = cstate->state; |
98fa08a5 | 1824 | struct regcache *reg_cache = cstate->reg_cache; |
cb72ea13 PC |
1825 | struct opcode *op = &block->opcode_list[offset]; |
1826 | bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay; | |
d16005f8 | 1827 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 1828 | jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2; |
9259d748 | 1829 | u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT; |
f5ee77ca | 1830 | bool different_offsets = state->offset_bios != state->offset_scratch; |
cb72ea13 | 1831 | union code c = op->c; |
684432ad PC |
1832 | s32 addr_mask; |
1833 | u32 reg_imm; | |
1834 | s8 offt_reg; | |
d16005f8 PC |
1835 | s16 imm; |
1836 | ||
cb72ea13 PC |
1837 | if (load_delay || c.i.op == OP_LWC2) |
1838 | out_reg = REG_TEMP; | |
9259d748 PC |
1839 | else if (c.i.rt) |
1840 | out_reg = c.i.rt; | |
1841 | else | |
d16005f8 PC |
1842 | return; |
1843 | ||
98fa08a5 PC |
1844 | if (is_unsigned) |
1845 | flags |= REG_ZEXT; | |
1846 | ||
d16005f8 | 1847 | jit_note(__FILE__, __LINE__); |
98fa08a5 | 1848 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0); |
9259d748 | 1849 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
d16005f8 PC |
1850 | |
1851 | if ((state->offset_ram == state->offset_bios && | |
1852 | state->offset_ram == state->offset_scratch && | |
5459088b PC |
1853 | state->mirrors_mapped && c.i.op != OP_META_LWU) |
1854 | || !c.i.imm) { | |
d16005f8 | 1855 | addr_reg = rs; |
98fa08a5 | 1856 | imm = (s16)c.i.imm; |
d16005f8 | 1857 | } else { |
98fa08a5 | 1858 | jit_addi(rt, rs, (s16)c.i.imm); |
d16005f8 PC |
1859 | addr_reg = rt; |
1860 | imm = 0; | |
1861 | ||
98fa08a5 | 1862 | if (c.i.rs != c.i.rt) |
d16005f8 PC |
1863 | lightrec_free_reg(reg_cache, rs); |
1864 | } | |
1865 | ||
5459088b PC |
1866 | if (op->i.op == OP_META_LWU) |
1867 | imm = LIGHTNING_UNALIGNED_32BIT; | |
1868 | ||
d16005f8 PC |
1869 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
1870 | ||
1871 | if (state->offset_ram == state->offset_bios && | |
1872 | state->offset_ram == state->offset_scratch) { | |
684432ad PC |
1873 | if (!state->mirrors_mapped) |
1874 | addr_mask = 0x1f800000 | (RAM_SIZE - 1); | |
1875 | else | |
1876 | addr_mask = 0x1fffffff; | |
1877 | ||
1878 | reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, | |
1879 | addr_mask); | |
d16005f8 PC |
1880 | if (!state->mirrors_mapped) { |
1881 | jit_andi(tmp, addr_reg, BIT(28)); | |
1882 | jit_rshi_u(tmp, tmp, 28 - 22); | |
684432ad | 1883 | jit_orr(tmp, tmp, reg_imm); |
d16005f8 PC |
1884 | jit_andr(rt, addr_reg, tmp); |
1885 | } else { | |
684432ad | 1886 | jit_andr(rt, addr_reg, reg_imm); |
d16005f8 PC |
1887 | } |
1888 | ||
684432ad PC |
1889 | lightrec_free_reg(reg_cache, reg_imm); |
1890 | ||
1891 | if (state->offset_ram) { | |
1892 | offt_reg = lightrec_get_reg_with_value(reg_cache, | |
1893 | state->offset_ram); | |
1894 | if (offt_reg < 0) { | |
1895 | jit_movi(tmp, state->offset_ram); | |
1896 | lightrec_temp_set_value(reg_cache, tmp, | |
1897 | state->offset_ram); | |
1898 | } else { | |
1899 | lightrec_free_reg(reg_cache, tmp); | |
1900 | tmp = offt_reg; | |
1901 | } | |
1902 | } | |
d16005f8 PC |
1903 | } else { |
1904 | to_not_ram = jit_bmsi(addr_reg, BIT(28)); | |
1905 | ||
1906 | /* Convert to KUNSEG and avoid RAM mirrors */ | |
1907 | jit_andi(rt, addr_reg, RAM_SIZE - 1); | |
1908 | ||
1909 | if (state->offset_ram) | |
1910 | jit_movi(tmp, state->offset_ram); | |
1911 | ||
22eee2ac | 1912 | to_end = jit_b(); |
d16005f8 PC |
1913 | |
1914 | jit_patch(to_not_ram); | |
1915 | ||
f5ee77ca | 1916 | if (different_offsets) |
d16005f8 PC |
1917 | to_not_bios = jit_bmci(addr_reg, BIT(22)); |
1918 | ||
1919 | /* Convert to KUNSEG */ | |
1920 | jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1)); | |
1921 | ||
1922 | jit_movi(tmp, state->offset_bios); | |
1923 | ||
f5ee77ca | 1924 | if (different_offsets) { |
22eee2ac | 1925 | to_end2 = jit_b(); |
d16005f8 PC |
1926 | |
1927 | jit_patch(to_not_bios); | |
1928 | ||
1929 | /* Convert to KUNSEG */ | |
1930 | jit_andi(rt, addr_reg, 0x1f800fff); | |
1931 | ||
1932 | if (state->offset_scratch) | |
1933 | jit_movi(tmp, state->offset_scratch); | |
1934 | ||
1935 | jit_patch(to_end2); | |
1936 | } | |
1937 | ||
1938 | jit_patch(to_end); | |
1939 | } | |
1940 | ||
1941 | if (state->offset_ram || state->offset_bios || state->offset_scratch) | |
1942 | jit_addr(rt, rt, tmp); | |
1943 | ||
1944 | jit_new_node_www(code, rt, rt, imm); | |
1945 | ||
02487de7 PC |
1946 | if (is_big_endian() && swap_code) { |
1947 | jit_new_node_ww(swap_code, rt, rt); | |
1948 | ||
1949 | if (c.i.op == OP_LH) | |
1950 | jit_extr_s(rt, rt); | |
1951 | else if (c.i.op == OP_LW && __WORDSIZE == 64) | |
1952 | jit_extr_i(rt, rt); | |
1953 | } | |
1954 | ||
d16005f8 PC |
1955 | lightrec_free_reg(reg_cache, addr_reg); |
1956 | lightrec_free_reg(reg_cache, rt); | |
1957 | lightrec_free_reg(reg_cache, tmp); | |
1958 | } | |
1959 | ||
98fa08a5 | 1960 | static void rec_load(struct lightrec_cstate *state, const struct block *block, |
02487de7 PC |
1961 | u16 offset, jit_code_t code, jit_code_t swap_code, |
1962 | bool is_unsigned) | |
d16005f8 | 1963 | { |
9259d748 PC |
1964 | const struct opcode *op = &block->opcode_list[offset]; |
1965 | u32 flags = op->flags; | |
98fa08a5 | 1966 | |
22eee2ac PC |
1967 | switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) { |
1968 | case LIGHTREC_IO_RAM: | |
02487de7 | 1969 | rec_load_ram(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1970 | break; |
1971 | case LIGHTREC_IO_BIOS: | |
02487de7 | 1972 | rec_load_bios(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1973 | break; |
1974 | case LIGHTREC_IO_SCRATCH: | |
02487de7 | 1975 | rec_load_scratch(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac | 1976 | break; |
ba3814c1 PC |
1977 | case LIGHTREC_IO_DIRECT_HW: |
1978 | rec_load_io(state, block, offset, code, swap_code, is_unsigned); | |
1979 | break; | |
22eee2ac | 1980 | case LIGHTREC_IO_DIRECT: |
02487de7 | 1981 | rec_load_direct(state, block, offset, code, swap_code, is_unsigned); |
22eee2ac PC |
1982 | break; |
1983 | default: | |
98fa08a5 | 1984 | rec_io(state, block, offset, false, true); |
9259d748 PC |
1985 | return; |
1986 | } | |
1987 | ||
1988 | if (op->i.op == OP_LWC2) { | |
cb72ea13 PC |
1989 | rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP); |
1990 | lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP); | |
22eee2ac | 1991 | } |
d16005f8 PC |
1992 | } |
1993 | ||
98fa08a5 | 1994 | static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
1995 | { |
1996 | _jit_name(block->_jit, __func__); | |
02487de7 | 1997 | rec_load(state, block, offset, jit_code_ldxi_c, 0, false); |
d16005f8 PC |
1998 | } |
1999 | ||
98fa08a5 | 2000 | static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
2001 | { |
2002 | _jit_name(block->_jit, __func__); | |
02487de7 | 2003 | rec_load(state, block, offset, jit_code_ldxi_uc, 0, true); |
d16005f8 PC |
2004 | } |
2005 | ||
98fa08a5 | 2006 | static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 2007 | { |
ba3814c1 PC |
2008 | jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s; |
2009 | ||
d16005f8 | 2010 | _jit_name(block->_jit, __func__); |
ba3814c1 | 2011 | rec_load(state, block, offset, code, jit_code_bswapr_us, false); |
d16005f8 PC |
2012 | } |
2013 | ||
98fa08a5 | 2014 | static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
2015 | { |
2016 | _jit_name(block->_jit, __func__); | |
02487de7 | 2017 | rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true); |
d16005f8 PC |
2018 | } |
2019 | ||
98fa08a5 | 2020 | static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
2021 | { |
2022 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2023 | rec_io(state, block, offset, true, true); |
d16005f8 PC |
2024 | } |
2025 | ||
98fa08a5 | 2026 | static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 PC |
2027 | { |
2028 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2029 | rec_io(state, block, offset, true, true); |
d16005f8 PC |
2030 | } |
2031 | ||
98fa08a5 | 2032 | static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset) |
d16005f8 | 2033 | { |
9259d748 | 2034 | union code c = block->opcode_list[offset].c; |
ba3814c1 PC |
2035 | jit_code_t code; |
2036 | ||
2037 | if (is_big_endian() && __WORDSIZE == 64) | |
2038 | code = jit_code_ldxi_ui; | |
2039 | else | |
2040 | code = jit_code_ldxi_i; | |
2041 | ||
9259d748 | 2042 | _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW"); |
ba3814c1 | 2043 | rec_load(state, block, offset, code, jit_code_bswapr_ui, false); |
d16005f8 PC |
2044 | } |
2045 | ||
e26c79a8 PC |
2046 | static void rec_exit_early(struct lightrec_cstate *state, |
2047 | const struct block *block, u16 offset, | |
2048 | u32 exit_code, u32 pc) | |
d16005f8 | 2049 | { |
ba3814c1 PC |
2050 | struct regcache *reg_cache = state->reg_cache; |
2051 | jit_state_t *_jit = block->_jit; | |
2052 | u8 tmp; | |
2053 | ||
98fa08a5 | 2054 | _jit_note(block->_jit, __FILE__, __LINE__); |
d16005f8 | 2055 | |
ba3814c1 PC |
2056 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
2057 | ||
2058 | jit_movi(tmp, exit_code); | |
8afce295 | 2059 | jit_stxi_i(lightrec_offset(exit_flags), LIGHTREC_REG_STATE, tmp); |
ba3814c1 | 2060 | |
8afce295 | 2061 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(target_cycle)); |
cb72ea13 PC |
2062 | jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE); |
2063 | jit_movi(LIGHTREC_REG_CYCLE, 0); | |
8afce295 PC |
2064 | jit_stxi_i(lightrec_offset(target_cycle), LIGHTREC_REG_STATE, tmp); |
2065 | jit_stxi_i(lightrec_offset(current_cycle), LIGHTREC_REG_STATE, tmp); | |
cb72ea13 | 2066 | |
ba3814c1 | 2067 | lightrec_free_reg(reg_cache, tmp); |
d16005f8 | 2068 | |
e26c79a8 | 2069 | lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true); |
d16005f8 PC |
2070 | } |
2071 | ||
98fa08a5 PC |
2072 | static void rec_special_SYSCALL(struct lightrec_cstate *state, |
2073 | const struct block *block, u16 offset) | |
d16005f8 PC |
2074 | { |
2075 | _jit_name(block->_jit, __func__); | |
e26c79a8 PC |
2076 | |
2077 | /* TODO: the return address should be "pc - 4" if we're a delay slot */ | |
2078 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL, | |
2079 | get_ds_pc(block, offset, 0)); | |
d16005f8 PC |
2080 | } |
2081 | ||
98fa08a5 PC |
2082 | static void rec_special_BREAK(struct lightrec_cstate *state, |
2083 | const struct block *block, u16 offset) | |
d16005f8 PC |
2084 | { |
2085 | _jit_name(block->_jit, __func__); | |
e26c79a8 PC |
2086 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK, |
2087 | get_ds_pc(block, offset, 0)); | |
d16005f8 PC |
2088 | } |
2089 | ||
fdf33147 PC |
2090 | static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset) |
2091 | { | |
2092 | struct regcache *reg_cache = state->reg_cache; | |
2093 | union code c = block->opcode_list[offset].c; | |
2094 | jit_state_t *_jit = block->_jit; | |
2095 | ||
2096 | jit_note(__FILE__, __LINE__); | |
9259d748 PC |
2097 | |
2098 | if (c.i.op != OP_SWC2) | |
2099 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true); | |
fdf33147 PC |
2100 | |
2101 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC); | |
2102 | } | |
2103 | ||
fd58fa32 | 2104 | static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset) |
98fa08a5 PC |
2105 | { |
2106 | struct regcache *reg_cache = state->reg_cache; | |
2107 | union code c = block->opcode_list[offset].c; | |
2108 | jit_state_t *_jit = block->_jit; | |
d16005f8 | 2109 | |
98fa08a5 PC |
2110 | jit_note(__FILE__, __LINE__); |
2111 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false); | |
2112 | lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false); | |
cb72ea13 | 2113 | lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false); |
d16005f8 | 2114 | |
ba3814c1 | 2115 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC); |
d16005f8 | 2116 | |
98fa08a5 | 2117 | if (c.i.op == OP_CP0 && |
03535202 | 2118 | !op_flag_no_ds(block->opcode_list[offset].flags) && |
98fa08a5 PC |
2119 | (c.r.rd == 12 || c.r.rd == 13)) |
2120 | lightrec_emit_end_of_block(state, block, offset, -1, | |
2121 | get_ds_pc(block, offset, 1), | |
2122 | 0, 0, true); | |
d16005f8 PC |
2123 | } |
2124 | ||
98fa08a5 PC |
2125 | static void |
2126 | rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
d16005f8 | 2127 | { |
d16005f8 | 2128 | struct regcache *reg_cache = state->reg_cache; |
98fa08a5 | 2129 | union code c = block->opcode_list[offset].c; |
d16005f8 | 2130 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 2131 | u8 rt; |
d16005f8 PC |
2132 | |
2133 | jit_note(__FILE__, __LINE__); | |
2134 | ||
98fa08a5 | 2135 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT); |
d16005f8 | 2136 | |
8afce295 | 2137 | jit_ldxi_i(rt, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[c.r.rd])); |
d16005f8 | 2138 | |
98fa08a5 PC |
2139 | lightrec_free_reg(reg_cache, rt); |
2140 | } | |
d16005f8 | 2141 | |
cb72ea13 PC |
2142 | static bool block_uses_icache(const struct lightrec_cstate *state, |
2143 | const struct block *block) | |
98fa08a5 | 2144 | { |
cb72ea13 | 2145 | const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM]; |
98fa08a5 | 2146 | u32 pc = kunseg(block->pc); |
d16005f8 | 2147 | |
cb72ea13 PC |
2148 | if (pc < map->pc || pc >= map->pc + map->length) |
2149 | return false; | |
2150 | ||
2151 | return (block->pc >> 28) < 0xa; | |
d16005f8 PC |
2152 | } |
2153 | ||
98fa08a5 PC |
2154 | static void |
2155 | rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset) | |
d16005f8 | 2156 | { |
98fa08a5 PC |
2157 | struct regcache *reg_cache = state->reg_cache; |
2158 | const union code c = block->opcode_list[offset].c; | |
2159 | jit_state_t *_jit = block->_jit; | |
fd58fa32 | 2160 | u8 rt, tmp = 0, tmp2, status; |
0cf41071 | 2161 | jit_node_t *to_end; |
98fa08a5 PC |
2162 | |
2163 | jit_note(__FILE__, __LINE__); | |
2164 | ||
2165 | switch(c.r.rd) { | |
2166 | case 1: | |
2167 | case 4: | |
2168 | case 8: | |
2169 | case 14: | |
2170 | case 15: | |
2171 | /* Those registers are read-only */ | |
2172 | return; | |
2173 | default: | |
2174 | break; | |
2175 | } | |
2176 | ||
cb72ea13 PC |
2177 | if (!block_uses_icache(state, block) && c.r.rd == 12) { |
2178 | /* If we are not running code from the RAM through kuseg or | |
2179 | * kseg0, handle writes to the Status register in C; as the | |
2180 | * code may toggle bit 16 which isolates the cache. Code | |
2181 | * running from kuseg or kseg0 in RAM cannot do that. */ | |
98fa08a5 PC |
2182 | rec_mtc(state, block, offset); |
2183 | return; | |
2184 | } | |
2185 | ||
2186 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0); | |
2187 | ||
8afce295 PC |
2188 | if (c.r.rd != 13) |
2189 | jit_stxi_i(lightrec_offset(regs.cp0[c.r.rd]), LIGHTREC_REG_STATE, rt); | |
98fa08a5 PC |
2190 | |
2191 | if (c.r.rd == 12 || c.r.rd == 13) { | |
2192 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
8afce295 | 2193 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[13])); |
fd58fa32 PC |
2194 | |
2195 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
98fa08a5 PC |
2196 | } |
2197 | ||
2198 | if (c.r.rd == 12) { | |
2199 | status = rt; | |
2200 | } else if (c.r.rd == 13) { | |
98fa08a5 PC |
2201 | /* Cause = (Cause & ~0x0300) | (value & 0x0300) */ |
2202 | jit_andi(tmp2, rt, 0x0300); | |
2203 | jit_ori(tmp, tmp, 0x0300); | |
2204 | jit_xori(tmp, tmp, 0x0300); | |
2205 | jit_orr(tmp, tmp, tmp2); | |
8afce295 PC |
2206 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[12])); |
2207 | jit_stxi_i(lightrec_offset(regs.cp0[13]), LIGHTREC_REG_STATE, tmp); | |
98fa08a5 PC |
2208 | status = tmp2; |
2209 | } | |
2210 | ||
2211 | if (c.r.rd == 12 || c.r.rd == 13) { | |
2212 | /* Exit dynarec in case there's a software interrupt. | |
2213 | * exit_flags = !!(status & tmp & 0x0300) & status; */ | |
2214 | jit_andr(tmp, tmp, status); | |
2215 | jit_andi(tmp, tmp, 0x0300); | |
2216 | jit_nei(tmp, tmp, 0); | |
2217 | jit_andr(tmp, tmp, status); | |
fd58fa32 PC |
2218 | } |
2219 | ||
2220 | if (c.r.rd == 12) { | |
2221 | /* Exit dynarec in case we unmask a hardware interrupt. | |
2222 | * exit_flags = !(~status & 0x401) */ | |
2223 | ||
2224 | jit_comr(tmp2, status); | |
2225 | jit_andi(tmp2, tmp2, 0x401); | |
2226 | jit_eqi(tmp2, tmp2, 0); | |
2227 | jit_orr(tmp, tmp, tmp2); | |
2228 | } | |
2229 | ||
0cf41071 PC |
2230 | lightrec_free_reg(reg_cache, rt); |
2231 | ||
fd58fa32 | 2232 | if (c.r.rd == 12 || c.r.rd == 13) { |
0cf41071 | 2233 | to_end = jit_beqi(tmp, 0); |
98fa08a5 | 2234 | |
8afce295 | 2235 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, lightrec_offset(target_cycle)); |
0cf41071 PC |
2236 | jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE); |
2237 | jit_movi(LIGHTREC_REG_CYCLE, 0); | |
8afce295 PC |
2238 | jit_stxi_i(lightrec_offset(target_cycle), LIGHTREC_REG_STATE, tmp2); |
2239 | jit_stxi_i(lightrec_offset(current_cycle), LIGHTREC_REG_STATE, tmp2); | |
98fa08a5 | 2240 | |
0cf41071 PC |
2241 | |
2242 | jit_patch(to_end); | |
2243 | } | |
98fa08a5 | 2244 | |
03535202 | 2245 | if (!op_flag_no_ds(block->opcode_list[offset].flags) && |
13b02197 | 2246 | (c.r.rd == 12 || c.r.rd == 13)) { |
684432ad | 2247 | state->cycles += lightrec_cycles_of_opcode(state->state, c); |
13b02197 PC |
2248 | lightrec_emit_eob(state, block, offset + 1); |
2249 | } | |
d16005f8 PC |
2250 | } |
2251 | ||
98fa08a5 PC |
2252 | static void rec_cp0_MFC0(struct lightrec_cstate *state, |
2253 | const struct block *block, u16 offset) | |
d16005f8 PC |
2254 | { |
2255 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2256 | rec_mfc0(state, block, offset); |
d16005f8 PC |
2257 | } |
2258 | ||
98fa08a5 PC |
2259 | static void rec_cp0_CFC0(struct lightrec_cstate *state, |
2260 | const struct block *block, u16 offset) | |
d16005f8 PC |
2261 | { |
2262 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2263 | rec_mfc0(state, block, offset); |
d16005f8 PC |
2264 | } |
2265 | ||
98fa08a5 PC |
2266 | static void rec_cp0_MTC0(struct lightrec_cstate *state, |
2267 | const struct block *block, u16 offset) | |
d16005f8 PC |
2268 | { |
2269 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2270 | rec_mtc0(state, block, offset); |
d16005f8 PC |
2271 | } |
2272 | ||
98fa08a5 PC |
2273 | static void rec_cp0_CTC0(struct lightrec_cstate *state, |
2274 | const struct block *block, u16 offset) | |
d16005f8 PC |
2275 | { |
2276 | _jit_name(block->_jit, __func__); | |
98fa08a5 | 2277 | rec_mtc0(state, block, offset); |
d16005f8 PC |
2278 | } |
2279 | ||
02487de7 PC |
2280 | static unsigned int cp2d_i_offset(u8 reg) |
2281 | { | |
8afce295 | 2282 | return lightrec_offset(regs.cp2d[reg]); |
02487de7 PC |
2283 | } |
2284 | ||
2285 | static unsigned int cp2d_s_offset(u8 reg) | |
2286 | { | |
2287 | return cp2d_i_offset(reg) + is_big_endian() * 2; | |
2288 | } | |
2289 | ||
2290 | static unsigned int cp2c_i_offset(u8 reg) | |
2291 | { | |
8afce295 | 2292 | return lightrec_offset(regs.cp2c[reg]); |
02487de7 PC |
2293 | } |
2294 | ||
2295 | static unsigned int cp2c_s_offset(u8 reg) | |
2296 | { | |
2297 | return cp2c_i_offset(reg) + is_big_endian() * 2; | |
2298 | } | |
2299 | ||
9259d748 PC |
2300 | static void rec_cp2_do_mfc2(struct lightrec_cstate *state, |
2301 | const struct block *block, u16 offset, | |
2302 | u8 reg, u8 out_reg) | |
d16005f8 | 2303 | { |
fd58fa32 | 2304 | struct regcache *reg_cache = state->reg_cache; |
fd58fa32 PC |
2305 | jit_state_t *_jit = block->_jit; |
2306 | const u32 zext_regs = 0x300f0080; | |
2307 | u8 rt, tmp, tmp2, tmp3, out, flags; | |
fd58fa32 PC |
2308 | unsigned int i; |
2309 | ||
d16005f8 | 2310 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2311 | |
fdf33147 PC |
2312 | if (state->state->ops.cop2_notify) { |
2313 | /* We must call cop2_notify, handle that in C. */ | |
2314 | rec_mfc(state, block, offset); | |
2315 | return; | |
2316 | } | |
2317 | ||
fd58fa32 | 2318 | flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT; |
9259d748 PC |
2319 | rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags); |
2320 | ||
2321 | if (reg == 15) | |
2322 | reg = 14; | |
fd58fa32 PC |
2323 | |
2324 | switch (reg) { | |
2325 | case 1: | |
2326 | case 3: | |
2327 | case 5: | |
2328 | case 8: | |
2329 | case 9: | |
2330 | case 10: | |
2331 | case 11: | |
02487de7 | 2332 | jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); |
fd58fa32 PC |
2333 | break; |
2334 | case 7: | |
2335 | case 16: | |
2336 | case 17: | |
2337 | case 18: | |
2338 | case 19: | |
02487de7 | 2339 | jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg)); |
fd58fa32 PC |
2340 | break; |
2341 | case 28: | |
2342 | case 29: | |
2343 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2344 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2345 | tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2346 | ||
2347 | for (i = 0; i < 3; i++) { | |
2348 | out = i == 0 ? rt : tmp; | |
2349 | ||
02487de7 | 2350 | jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i)); |
fd58fa32 PC |
2351 | jit_movi(tmp2, 0x1f); |
2352 | jit_rshi(out, tmp, 7); | |
2353 | ||
2354 | jit_ltr(tmp3, tmp2, out); | |
2355 | jit_movnr(out, tmp2, tmp3); | |
2356 | ||
2357 | jit_gei(tmp2, out, 0); | |
2358 | jit_movzr(out, tmp2, tmp2); | |
2359 | ||
2360 | if (i > 0) { | |
2361 | jit_lshi(tmp, tmp, 5 * i); | |
2362 | jit_orr(rt, rt, tmp); | |
2363 | } | |
2364 | } | |
2365 | ||
2366 | ||
2367 | lightrec_free_reg(reg_cache, tmp); | |
2368 | lightrec_free_reg(reg_cache, tmp2); | |
2369 | lightrec_free_reg(reg_cache, tmp3); | |
2370 | break; | |
2371 | default: | |
02487de7 | 2372 | jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg)); |
fd58fa32 PC |
2373 | break; |
2374 | } | |
2375 | ||
2376 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2377 | } |
2378 | ||
9259d748 PC |
2379 | static void rec_cp2_basic_MFC2(struct lightrec_cstate *state, |
2380 | const struct block *block, u16 offset) | |
2381 | { | |
2382 | const union code c = block->opcode_list[offset].c; | |
2383 | ||
2384 | rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt); | |
2385 | } | |
2386 | ||
98fa08a5 PC |
2387 | static void rec_cp2_basic_CFC2(struct lightrec_cstate *state, |
2388 | const struct block *block, u16 offset) | |
d16005f8 | 2389 | { |
fd58fa32 PC |
2390 | struct regcache *reg_cache = state->reg_cache; |
2391 | const union code c = block->opcode_list[offset].c; | |
2392 | jit_state_t *_jit = block->_jit; | |
2393 | u8 rt; | |
2394 | ||
d16005f8 | 2395 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2396 | |
fdf33147 PC |
2397 | if (state->state->ops.cop2_notify) { |
2398 | /* We must call cop2_notify, handle that in C. */ | |
2399 | rec_mfc(state, block, offset); | |
2400 | return; | |
2401 | } | |
2402 | ||
fd58fa32 PC |
2403 | switch (c.r.rd) { |
2404 | case 4: | |
2405 | case 12: | |
2406 | case 20: | |
2407 | case 26: | |
2408 | case 27: | |
2409 | case 29: | |
2410 | case 30: | |
2411 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT); | |
02487de7 | 2412 | jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd)); |
fd58fa32 PC |
2413 | break; |
2414 | default: | |
2415 | rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT); | |
0cf41071 | 2416 | jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd)); |
fd58fa32 PC |
2417 | break; |
2418 | } | |
2419 | ||
2420 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2421 | } |
2422 | ||
9259d748 PC |
2423 | static void rec_cp2_do_mtc2(struct lightrec_cstate *state, |
2424 | const struct block *block, u16 offset, | |
2425 | u8 reg, u8 in_reg) | |
d16005f8 | 2426 | { |
fd58fa32 | 2427 | struct regcache *reg_cache = state->reg_cache; |
fd58fa32 | 2428 | jit_state_t *_jit = block->_jit; |
fd58fa32 PC |
2429 | u8 rt, tmp, tmp2, flags = 0; |
2430 | ||
d16005f8 | 2431 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2432 | |
fdf33147 PC |
2433 | if (state->state->ops.cop2_notify) { |
2434 | /* We must call cop2_notify, handle that in C. */ | |
2435 | rec_mtc(state, block, offset); | |
2436 | return; | |
2437 | } | |
2438 | ||
9259d748 | 2439 | if (reg == 31) |
fd58fa32 PC |
2440 | return; |
2441 | ||
9259d748 | 2442 | if (reg == 30) |
fd58fa32 PC |
2443 | flags |= REG_EXT; |
2444 | ||
9259d748 | 2445 | rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags); |
fd58fa32 | 2446 | |
9259d748 | 2447 | switch (reg) { |
fd58fa32 PC |
2448 | case 15: |
2449 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
02487de7 | 2450 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13)); |
fd58fa32 PC |
2451 | |
2452 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
02487de7 | 2453 | jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14)); |
fd58fa32 | 2454 | |
02487de7 PC |
2455 | jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp); |
2456 | jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2); | |
2457 | jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt); | |
fd58fa32 PC |
2458 | |
2459 | lightrec_free_reg(reg_cache, tmp); | |
2460 | lightrec_free_reg(reg_cache, tmp2); | |
2461 | break; | |
2462 | case 28: | |
2463 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2464 | ||
2465 | jit_lshi(tmp, rt, 7); | |
2466 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2467 | jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2468 | |
2469 | jit_lshi(tmp, rt, 2); | |
2470 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2471 | jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2472 | |
2473 | jit_rshi(tmp, rt, 3); | |
2474 | jit_andi(tmp, tmp, 0xf80); | |
02487de7 | 2475 | jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2476 | |
2477 | lightrec_free_reg(reg_cache, tmp); | |
2478 | break; | |
2479 | case 30: | |
2480 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
fd58fa32 PC |
2481 | |
2482 | /* if (rt < 0) rt = ~rt; */ | |
2483 | jit_rshi(tmp, rt, 31); | |
2484 | jit_xorr(tmp, rt, tmp); | |
2485 | ||
cb72ea13 PC |
2486 | /* Count leading zeros */ |
2487 | jit_clzr(tmp, tmp); | |
2488 | if (__WORDSIZE != 32) | |
2489 | jit_subi(tmp, tmp, __WORDSIZE - 32); | |
fd58fa32 | 2490 | |
cb72ea13 | 2491 | jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2492 | |
2493 | lightrec_free_reg(reg_cache, tmp); | |
cb72ea13 | 2494 | fallthrough; |
fd58fa32 | 2495 | default: |
9259d748 | 2496 | jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2497 | break; |
2498 | } | |
2499 | ||
2500 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2501 | } |
2502 | ||
9259d748 PC |
2503 | static void rec_cp2_basic_MTC2(struct lightrec_cstate *state, |
2504 | const struct block *block, u16 offset) | |
2505 | { | |
2506 | const union code c = block->opcode_list[offset].c; | |
2507 | ||
2508 | rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt); | |
2509 | } | |
2510 | ||
98fa08a5 PC |
2511 | static void rec_cp2_basic_CTC2(struct lightrec_cstate *state, |
2512 | const struct block *block, u16 offset) | |
d16005f8 | 2513 | { |
fd58fa32 PC |
2514 | struct regcache *reg_cache = state->reg_cache; |
2515 | const union code c = block->opcode_list[offset].c; | |
2516 | jit_state_t *_jit = block->_jit; | |
2517 | u8 rt, tmp, tmp2; | |
2518 | ||
98fa08a5 | 2519 | _jit_name(block->_jit, __func__); |
fd58fa32 | 2520 | |
fdf33147 PC |
2521 | if (state->state->ops.cop2_notify) { |
2522 | /* We must call cop2_notify, handle that in C. */ | |
2523 | rec_mtc(state, block, offset); | |
2524 | return; | |
2525 | } | |
2526 | ||
fd58fa32 PC |
2527 | rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0); |
2528 | ||
2529 | switch (c.r.rd) { | |
2530 | case 4: | |
2531 | case 12: | |
2532 | case 20: | |
2533 | case 26: | |
2534 | case 27: | |
2535 | case 29: | |
2536 | case 30: | |
02487de7 | 2537 | jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2538 | break; |
2539 | case 31: | |
2540 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2541 | tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit); | |
2542 | ||
2543 | jit_andi(tmp, rt, 0x7f87e000); | |
2544 | jit_nei(tmp, tmp, 0); | |
2545 | jit_lshi(tmp, tmp, 31); | |
2546 | ||
2547 | jit_andi(tmp2, rt, 0x7ffff000); | |
2548 | jit_orr(tmp, tmp2, tmp); | |
2549 | ||
02487de7 | 2550 | jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp); |
fd58fa32 PC |
2551 | |
2552 | lightrec_free_reg(reg_cache, tmp); | |
2553 | lightrec_free_reg(reg_cache, tmp2); | |
2554 | break; | |
2555 | ||
2556 | default: | |
02487de7 | 2557 | jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt); |
fd58fa32 PC |
2558 | } |
2559 | ||
2560 | lightrec_free_reg(reg_cache, rt); | |
d16005f8 PC |
2561 | } |
2562 | ||
98fa08a5 PC |
2563 | static void rec_cp0_RFE(struct lightrec_cstate *state, |
2564 | const struct block *block, u16 offset) | |
d16005f8 | 2565 | { |
98fa08a5 | 2566 | struct regcache *reg_cache = state->reg_cache; |
d16005f8 | 2567 | jit_state_t *_jit = block->_jit; |
98fa08a5 | 2568 | u8 status, tmp; |
d16005f8 PC |
2569 | |
2570 | jit_name(__func__); | |
2571 | jit_note(__FILE__, __LINE__); | |
2572 | ||
98fa08a5 | 2573 | status = lightrec_alloc_reg_temp(reg_cache, _jit); |
8afce295 | 2574 | jit_ldxi_i(status, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[12])); |
d16005f8 | 2575 | |
98fa08a5 | 2576 | tmp = lightrec_alloc_reg_temp(reg_cache, _jit); |
d16005f8 | 2577 | |
98fa08a5 PC |
2578 | /* status = ((status >> 2) & 0xf) | status & ~0xf; */ |
2579 | jit_rshi(tmp, status, 2); | |
2580 | jit_andi(tmp, tmp, 0xf); | |
2581 | jit_andi(status, status, ~0xful); | |
2582 | jit_orr(status, status, tmp); | |
2583 | ||
8afce295 PC |
2584 | jit_ldxi_i(tmp, LIGHTREC_REG_STATE, lightrec_offset(regs.cp0[13])); |
2585 | jit_stxi_i(lightrec_offset(regs.cp0[12]), LIGHTREC_REG_STATE, status); | |
98fa08a5 PC |
2586 | |
2587 | /* Exit dynarec in case there's a software interrupt. | |
2588 | * exit_flags = !!(status & cause & 0x0300) & status; */ | |
2589 | jit_andr(tmp, tmp, status); | |
2590 | jit_andi(tmp, tmp, 0x0300); | |
2591 | jit_nei(tmp, tmp, 0); | |
2592 | jit_andr(tmp, tmp, status); | |
8afce295 | 2593 | jit_stxi_i(lightrec_offset(exit_flags), LIGHTREC_REG_STATE, tmp); |
98fa08a5 PC |
2594 | |
2595 | lightrec_free_reg(reg_cache, status); | |
d16005f8 | 2596 | lightrec_free_reg(reg_cache, tmp); |
d16005f8 PC |
2597 | } |
2598 | ||
98fa08a5 PC |
2599 | static void rec_CP(struct lightrec_cstate *state, |
2600 | const struct block *block, u16 offset) | |
d16005f8 | 2601 | { |
98fa08a5 | 2602 | union code c = block->opcode_list[offset].c; |
d16005f8 PC |
2603 | jit_state_t *_jit = block->_jit; |
2604 | ||
2605 | jit_name(__func__); | |
2606 | jit_note(__FILE__, __LINE__); | |
2607 | ||
ba3814c1 | 2608 | call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP); |
d16005f8 PC |
2609 | } |
2610 | ||
98fa08a5 PC |
2611 | static void rec_meta_MOV(struct lightrec_cstate *state, |
2612 | const struct block *block, u16 offset) | |
d16005f8 | 2613 | { |
d16005f8 | 2614 | struct regcache *reg_cache = state->reg_cache; |
9259d748 PC |
2615 | const struct opcode *op = &block->opcode_list[offset]; |
2616 | union code c = op->c; | |
d16005f8 | 2617 | jit_state_t *_jit = block->_jit; |
9259d748 | 2618 | bool unload_rd; |
684432ad | 2619 | bool unload_rs, discard_rs; |
d16005f8 PC |
2620 | u8 rs, rd; |
2621 | ||
2622 | _jit_name(block->_jit, __func__); | |
2623 | jit_note(__FILE__, __LINE__); | |
9259d748 | 2624 | |
684432ad PC |
2625 | unload_rs = OPT_EARLY_UNLOAD |
2626 | && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD; | |
2627 | discard_rs = OPT_EARLY_UNLOAD | |
2628 | && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD; | |
2629 | ||
2630 | if ((unload_rs || discard_rs) && c.m.rs) { | |
2631 | /* If the source register is going to be unloaded or discarded, | |
2632 | * then we can simply mark its host register as now pointing to | |
2633 | * the destination register. */ | |
2634 | pr_debug("Remap %s to %s at offset 0x%x\n", | |
2635 | lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd), | |
2636 | offset << 2); | |
2637 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
2638 | lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs); | |
2639 | lightrec_free_reg(reg_cache, rs); | |
2640 | return; | |
2641 | } | |
2642 | ||
9259d748 PC |
2643 | unload_rd = OPT_EARLY_UNLOAD |
2644 | && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD; | |
2645 | ||
8afce295 | 2646 | if (unload_rd) { |
9259d748 PC |
2647 | /* If the destination register will be unloaded right after the |
2648 | * MOV meta-opcode, we don't actually need to write any host | |
2649 | * register - we can just store the source register directly to | |
2650 | * the register cache, at the offset corresponding to the | |
2651 | * destination register. */ | |
cb72ea13 PC |
2652 | lightrec_discard_reg_if_loaded(reg_cache, c.m.rd); |
2653 | ||
2654 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
9259d748 | 2655 | |
8afce295 | 2656 | jit_stxi_i(lightrec_offset(regs.gpr) + (c.m.rd << 2), LIGHTREC_REG_STATE, rs); |
d16005f8 | 2657 | |
98fa08a5 | 2658 | lightrec_free_reg(reg_cache, rs); |
9259d748 | 2659 | } else { |
cb72ea13 PC |
2660 | if (c.m.rs) |
2661 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0); | |
2662 | ||
2663 | rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT); | |
9259d748 | 2664 | |
cb72ea13 | 2665 | if (c.m.rs == 0) { |
9259d748 | 2666 | jit_movi(rd, 0); |
cb72ea13 | 2667 | } else { |
9259d748 | 2668 | jit_extr_i(rd, rs); |
cb72ea13 PC |
2669 | lightrec_free_reg(reg_cache, rs); |
2670 | } | |
9259d748 PC |
2671 | |
2672 | lightrec_free_reg(reg_cache, rd); | |
2673 | } | |
d16005f8 PC |
2674 | } |
2675 | ||
98fa08a5 PC |
2676 | static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state, |
2677 | const struct block *block, | |
2678 | u16 offset) | |
d16005f8 | 2679 | { |
98fa08a5 PC |
2680 | struct regcache *reg_cache = state->reg_cache; |
2681 | union code c = block->opcode_list[offset].c; | |
d16005f8 | 2682 | jit_state_t *_jit = block->_jit; |
cb72ea13 | 2683 | u8 rs, rd; |
d16005f8 | 2684 | |
98fa08a5 | 2685 | _jit_name(block->_jit, __func__); |
d16005f8 PC |
2686 | jit_note(__FILE__, __LINE__); |
2687 | ||
684432ad PC |
2688 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], |
2689 | c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd); | |
d16005f8 | 2690 | |
cb72ea13 PC |
2691 | if (c.m.op == OP_META_EXTC) |
2692 | jit_extr_c(rd, rs); | |
98fa08a5 | 2693 | else |
cb72ea13 | 2694 | jit_extr_s(rd, rs); |
d16005f8 | 2695 | |
98fa08a5 | 2696 | lightrec_free_reg(reg_cache, rs); |
cb72ea13 | 2697 | lightrec_free_reg(reg_cache, rd); |
d16005f8 PC |
2698 | } |
2699 | ||
ba3814c1 PC |
2700 | static void rec_meta_MULT2(struct lightrec_cstate *state, |
2701 | const struct block *block, | |
2702 | u16 offset) | |
2703 | { | |
2704 | struct regcache *reg_cache = state->reg_cache; | |
2705 | union code c = block->opcode_list[offset].c; | |
2706 | jit_state_t *_jit = block->_jit; | |
2707 | u8 reg_lo = get_mult_div_lo(c); | |
2708 | u8 reg_hi = get_mult_div_hi(c); | |
2709 | u32 flags = block->opcode_list[offset].flags; | |
2710 | bool is_signed = c.i.op == OP_META_MULT2; | |
2711 | u8 rs, lo, hi, rflags = 0, hiflags = 0; | |
0cf41071 | 2712 | unsigned int i; |
ba3814c1 PC |
2713 | |
2714 | if (!op_flag_no_hi(flags) && c.r.op < 32) { | |
2715 | rflags = is_signed ? REG_EXT : REG_ZEXT; | |
2716 | hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT); | |
2717 | } | |
2718 | ||
2719 | _jit_name(block->_jit, __func__); | |
2720 | jit_note(__FILE__, __LINE__); | |
2721 | ||
2722 | rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags); | |
2723 | ||
0cf41071 PC |
2724 | /* |
2725 | * We must handle the case where one of the output registers is our rs | |
2726 | * input register. Thanksfully, computing LO/HI can be done in any | |
2727 | * order. Here, we make sure that the computation that overwrites the | |
2728 | * input register is always performed last. | |
2729 | */ | |
2730 | for (i = 0; i < 2; i++) { | |
2731 | if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) { | |
2732 | lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0); | |
2733 | ||
2734 | if (c.r.op < 32) | |
2735 | jit_lshi(lo, rs, c.r.op); | |
2736 | else | |
2737 | jit_movi(lo, 0); | |
ba3814c1 | 2738 | |
0cf41071 PC |
2739 | lightrec_free_reg(reg_cache, lo); |
2740 | continue; | |
2741 | } | |
ba3814c1 | 2742 | |
0cf41071 PC |
2743 | if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) { |
2744 | hi = lightrec_alloc_reg_out(reg_cache, _jit, | |
2745 | reg_hi, hiflags); | |
ba3814c1 | 2746 | |
f5ee77ca | 2747 | if (c.r.op >= 32) { |
0cf41071 | 2748 | jit_lshi(hi, rs, c.r.op - 32); |
f5ee77ca PC |
2749 | } else if (is_signed) { |
2750 | if (c.r.op) | |
2751 | jit_rshi(hi, rs, 32 - c.r.op); | |
2752 | else | |
2753 | jit_rshi(hi, rs, 31); | |
2754 | } else { | |
2755 | if (c.r.op) | |
2756 | jit_rshi_u(hi, rs, 32 - c.r.op); | |
2757 | else | |
2758 | jit_movi(hi, 0); | |
2759 | } | |
ba3814c1 | 2760 | |
0cf41071 PC |
2761 | lightrec_free_reg(reg_cache, hi); |
2762 | } | |
ba3814c1 PC |
2763 | } |
2764 | ||
2765 | lightrec_free_reg(reg_cache, rs); | |
2766 | ||
2767 | _jit_name(block->_jit, __func__); | |
2768 | jit_note(__FILE__, __LINE__); | |
2769 | } | |
2770 | ||
cb72ea13 PC |
2771 | static void rec_meta_COM(struct lightrec_cstate *state, |
2772 | const struct block *block, u16 offset) | |
2773 | { | |
2774 | struct regcache *reg_cache = state->reg_cache; | |
2775 | union code c = block->opcode_list[offset].c; | |
2776 | jit_state_t *_jit = block->_jit; | |
2777 | u8 rd, rs, flags; | |
2778 | ||
2779 | jit_note(__FILE__, __LINE__); | |
684432ad PC |
2780 | |
2781 | rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset], | |
2782 | c.m.rs, c.m.rd, 0, 0, &rs, &rd); | |
cb72ea13 PC |
2783 | |
2784 | flags = lightrec_get_reg_in_flags(reg_cache, rs); | |
2785 | ||
2786 | lightrec_set_reg_out_flags(reg_cache, rd, | |
2787 | flags & REG_EXT); | |
2788 | ||
2789 | jit_comr(rd, rs); | |
2790 | ||
2791 | lightrec_free_reg(reg_cache, rs); | |
2792 | lightrec_free_reg(reg_cache, rd); | |
2793 | } | |
2794 | ||
5459088b PC |
2795 | static void rec_meta_LWU(struct lightrec_cstate *state, |
2796 | const struct block *block, | |
2797 | u16 offset) | |
2798 | { | |
2799 | jit_code_t code; | |
2800 | ||
2801 | if (is_big_endian() && __WORDSIZE == 64) | |
2802 | code = jit_code_unldr_u; | |
2803 | else | |
2804 | code = jit_code_unldr; | |
2805 | ||
2806 | _jit_name(block->_jit, __func__); | |
2807 | rec_load(state, block, offset, code, jit_code_bswapr_ui, false); | |
2808 | } | |
2809 | ||
2810 | static void rec_meta_SWU(struct lightrec_cstate *state, | |
2811 | const struct block *block, | |
2812 | u16 offset) | |
2813 | { | |
2814 | _jit_name(block->_jit, __func__); | |
2815 | rec_store(state, block, offset, jit_code_unstr, jit_code_bswapr_ui); | |
2816 | } | |
2817 | ||
e26c79a8 PC |
2818 | static void unknown_opcode(struct lightrec_cstate *state, |
2819 | const struct block *block, u16 offset) | |
2820 | { | |
2821 | rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP, | |
2822 | block->pc + (offset << 2)); | |
2823 | } | |
2824 | ||
d16005f8 | 2825 | static const lightrec_rec_func_t rec_standard[64] = { |
98fa08a5 | 2826 | SET_DEFAULT_ELM(rec_standard, unknown_opcode), |
d16005f8 PC |
2827 | [OP_SPECIAL] = rec_SPECIAL, |
2828 | [OP_REGIMM] = rec_REGIMM, | |
2829 | [OP_J] = rec_J, | |
2830 | [OP_JAL] = rec_JAL, | |
2831 | [OP_BEQ] = rec_BEQ, | |
2832 | [OP_BNE] = rec_BNE, | |
2833 | [OP_BLEZ] = rec_BLEZ, | |
2834 | [OP_BGTZ] = rec_BGTZ, | |
2835 | [OP_ADDI] = rec_ADDI, | |
2836 | [OP_ADDIU] = rec_ADDIU, | |
2837 | [OP_SLTI] = rec_SLTI, | |
2838 | [OP_SLTIU] = rec_SLTIU, | |
2839 | [OP_ANDI] = rec_ANDI, | |
2840 | [OP_ORI] = rec_ORI, | |
2841 | [OP_XORI] = rec_XORI, | |
2842 | [OP_LUI] = rec_LUI, | |
2843 | [OP_CP0] = rec_CP0, | |
2844 | [OP_CP2] = rec_CP2, | |
2845 | [OP_LB] = rec_LB, | |
2846 | [OP_LH] = rec_LH, | |
2847 | [OP_LWL] = rec_LWL, | |
2848 | [OP_LW] = rec_LW, | |
2849 | [OP_LBU] = rec_LBU, | |
2850 | [OP_LHU] = rec_LHU, | |
2851 | [OP_LWR] = rec_LWR, | |
2852 | [OP_SB] = rec_SB, | |
2853 | [OP_SH] = rec_SH, | |
2854 | [OP_SWL] = rec_SWL, | |
2855 | [OP_SW] = rec_SW, | |
2856 | [OP_SWR] = rec_SWR, | |
9259d748 PC |
2857 | [OP_LWC2] = rec_LW, |
2858 | [OP_SWC2] = rec_SW, | |
d16005f8 | 2859 | |
cb72ea13 | 2860 | [OP_META] = rec_META, |
ba3814c1 PC |
2861 | [OP_META_MULT2] = rec_meta_MULT2, |
2862 | [OP_META_MULTU2] = rec_meta_MULT2, | |
5459088b PC |
2863 | [OP_META_LWU] = rec_meta_LWU, |
2864 | [OP_META_SWU] = rec_meta_SWU, | |
d16005f8 PC |
2865 | }; |
2866 | ||
2867 | static const lightrec_rec_func_t rec_special[64] = { | |
98fa08a5 | 2868 | SET_DEFAULT_ELM(rec_special, unknown_opcode), |
d16005f8 PC |
2869 | [OP_SPECIAL_SLL] = rec_special_SLL, |
2870 | [OP_SPECIAL_SRL] = rec_special_SRL, | |
2871 | [OP_SPECIAL_SRA] = rec_special_SRA, | |
2872 | [OP_SPECIAL_SLLV] = rec_special_SLLV, | |
2873 | [OP_SPECIAL_SRLV] = rec_special_SRLV, | |
2874 | [OP_SPECIAL_SRAV] = rec_special_SRAV, | |
2875 | [OP_SPECIAL_JR] = rec_special_JR, | |
2876 | [OP_SPECIAL_JALR] = rec_special_JALR, | |
2877 | [OP_SPECIAL_SYSCALL] = rec_special_SYSCALL, | |
2878 | [OP_SPECIAL_BREAK] = rec_special_BREAK, | |
2879 | [OP_SPECIAL_MFHI] = rec_special_MFHI, | |
2880 | [OP_SPECIAL_MTHI] = rec_special_MTHI, | |
2881 | [OP_SPECIAL_MFLO] = rec_special_MFLO, | |
2882 | [OP_SPECIAL_MTLO] = rec_special_MTLO, | |
2883 | [OP_SPECIAL_MULT] = rec_special_MULT, | |
2884 | [OP_SPECIAL_MULTU] = rec_special_MULTU, | |
2885 | [OP_SPECIAL_DIV] = rec_special_DIV, | |
2886 | [OP_SPECIAL_DIVU] = rec_special_DIVU, | |
2887 | [OP_SPECIAL_ADD] = rec_special_ADD, | |
2888 | [OP_SPECIAL_ADDU] = rec_special_ADDU, | |
2889 | [OP_SPECIAL_SUB] = rec_special_SUB, | |
2890 | [OP_SPECIAL_SUBU] = rec_special_SUBU, | |
2891 | [OP_SPECIAL_AND] = rec_special_AND, | |
2892 | [OP_SPECIAL_OR] = rec_special_OR, | |
2893 | [OP_SPECIAL_XOR] = rec_special_XOR, | |
2894 | [OP_SPECIAL_NOR] = rec_special_NOR, | |
2895 | [OP_SPECIAL_SLT] = rec_special_SLT, | |
2896 | [OP_SPECIAL_SLTU] = rec_special_SLTU, | |
2897 | }; | |
2898 | ||
2899 | static const lightrec_rec_func_t rec_regimm[64] = { | |
98fa08a5 | 2900 | SET_DEFAULT_ELM(rec_regimm, unknown_opcode), |
d16005f8 PC |
2901 | [OP_REGIMM_BLTZ] = rec_regimm_BLTZ, |
2902 | [OP_REGIMM_BGEZ] = rec_regimm_BGEZ, | |
2903 | [OP_REGIMM_BLTZAL] = rec_regimm_BLTZAL, | |
2904 | [OP_REGIMM_BGEZAL] = rec_regimm_BGEZAL, | |
2905 | }; | |
2906 | ||
2907 | static const lightrec_rec_func_t rec_cp0[64] = { | |
98fa08a5 | 2908 | SET_DEFAULT_ELM(rec_cp0, rec_CP), |
d16005f8 PC |
2909 | [OP_CP0_MFC0] = rec_cp0_MFC0, |
2910 | [OP_CP0_CFC0] = rec_cp0_CFC0, | |
2911 | [OP_CP0_MTC0] = rec_cp0_MTC0, | |
2912 | [OP_CP0_CTC0] = rec_cp0_CTC0, | |
2913 | [OP_CP0_RFE] = rec_cp0_RFE, | |
2914 | }; | |
2915 | ||
2916 | static const lightrec_rec_func_t rec_cp2_basic[64] = { | |
98fa08a5 | 2917 | SET_DEFAULT_ELM(rec_cp2_basic, rec_CP), |
d16005f8 PC |
2918 | [OP_CP2_BASIC_MFC2] = rec_cp2_basic_MFC2, |
2919 | [OP_CP2_BASIC_CFC2] = rec_cp2_basic_CFC2, | |
2920 | [OP_CP2_BASIC_MTC2] = rec_cp2_basic_MTC2, | |
2921 | [OP_CP2_BASIC_CTC2] = rec_cp2_basic_CTC2, | |
2922 | }; | |
2923 | ||
cb72ea13 PC |
2924 | static const lightrec_rec_func_t rec_meta[64] = { |
2925 | SET_DEFAULT_ELM(rec_meta, unknown_opcode), | |
2926 | [OP_META_MOV] = rec_meta_MOV, | |
2927 | [OP_META_EXTC] = rec_meta_EXTC_EXTS, | |
2928 | [OP_META_EXTS] = rec_meta_EXTC_EXTS, | |
2929 | [OP_META_COM] = rec_meta_COM, | |
2930 | }; | |
2931 | ||
98fa08a5 PC |
2932 | static void rec_SPECIAL(struct lightrec_cstate *state, |
2933 | const struct block *block, u16 offset) | |
d16005f8 | 2934 | { |
98fa08a5 PC |
2935 | union code c = block->opcode_list[offset].c; |
2936 | lightrec_rec_func_t f = rec_special[c.r.op]; | |
2937 | ||
2938 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2939 | unknown_opcode(state, block, offset); | |
d16005f8 | 2940 | else |
98fa08a5 | 2941 | (*f)(state, block, offset); |
d16005f8 PC |
2942 | } |
2943 | ||
98fa08a5 PC |
2944 | static void rec_REGIMM(struct lightrec_cstate *state, |
2945 | const struct block *block, u16 offset) | |
d16005f8 | 2946 | { |
98fa08a5 PC |
2947 | union code c = block->opcode_list[offset].c; |
2948 | lightrec_rec_func_t f = rec_regimm[c.r.rt]; | |
2949 | ||
2950 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2951 | unknown_opcode(state, block, offset); | |
d16005f8 | 2952 | else |
98fa08a5 | 2953 | (*f)(state, block, offset); |
d16005f8 PC |
2954 | } |
2955 | ||
98fa08a5 PC |
2956 | static void rec_CP0(struct lightrec_cstate *state, |
2957 | const struct block *block, u16 offset) | |
d16005f8 | 2958 | { |
98fa08a5 PC |
2959 | union code c = block->opcode_list[offset].c; |
2960 | lightrec_rec_func_t f = rec_cp0[c.r.rs]; | |
2961 | ||
2962 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2963 | rec_CP(state, block, offset); | |
d16005f8 | 2964 | else |
98fa08a5 | 2965 | (*f)(state, block, offset); |
d16005f8 PC |
2966 | } |
2967 | ||
98fa08a5 PC |
2968 | static void rec_CP2(struct lightrec_cstate *state, |
2969 | const struct block *block, u16 offset) | |
d16005f8 | 2970 | { |
98fa08a5 PC |
2971 | union code c = block->opcode_list[offset].c; |
2972 | ||
2973 | if (c.r.op == OP_CP2_BASIC) { | |
2974 | lightrec_rec_func_t f = rec_cp2_basic[c.r.rs]; | |
2975 | ||
2976 | if (HAS_DEFAULT_ELM || likely(f)) { | |
2977 | (*f)(state, block, offset); | |
d16005f8 PC |
2978 | return; |
2979 | } | |
2980 | } | |
2981 | ||
98fa08a5 | 2982 | rec_CP(state, block, offset); |
d16005f8 PC |
2983 | } |
2984 | ||
cb72ea13 PC |
2985 | static void rec_META(struct lightrec_cstate *state, |
2986 | const struct block *block, u16 offset) | |
2987 | { | |
2988 | union code c = block->opcode_list[offset].c; | |
2989 | lightrec_rec_func_t f = rec_meta[c.m.op]; | |
2990 | ||
2991 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
2992 | unknown_opcode(state, block, offset); | |
2993 | else | |
2994 | (*f)(state, block, offset); | |
2995 | } | |
2996 | ||
98fa08a5 PC |
2997 | void lightrec_rec_opcode(struct lightrec_cstate *state, |
2998 | const struct block *block, u16 offset) | |
d16005f8 | 2999 | { |
98fa08a5 PC |
3000 | struct regcache *reg_cache = state->reg_cache; |
3001 | struct lightrec_branch_target *target; | |
3002 | const struct opcode *op = &block->opcode_list[offset]; | |
3003 | jit_state_t *_jit = block->_jit; | |
3004 | lightrec_rec_func_t f; | |
03535202 | 3005 | u16 unload_offset; |
98fa08a5 | 3006 | |
03535202 PC |
3007 | if (op_flag_sync(op->flags)) { |
3008 | if (state->cycles) | |
3009 | jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles); | |
98fa08a5 PC |
3010 | state->cycles = 0; |
3011 | ||
3012 | lightrec_storeback_regs(reg_cache, _jit); | |
3013 | lightrec_regcache_reset(reg_cache); | |
3014 | ||
3015 | pr_debug("Adding branch target at offset 0x%x\n", offset << 2); | |
3016 | target = &state->targets[state->nb_targets++]; | |
3017 | target->offset = offset; | |
3018 | target->label = jit_indirect(); | |
3019 | } | |
3020 | ||
3021 | if (likely(op->opcode)) { | |
3022 | f = rec_standard[op->i.op]; | |
3023 | ||
3024 | if (!HAS_DEFAULT_ELM && unlikely(!f)) | |
3025 | unknown_opcode(state, block, offset); | |
3026 | else | |
3027 | (*f)(state, block, offset); | |
3028 | } | |
3029 | ||
03535202 PC |
3030 | if (OPT_EARLY_UNLOAD) { |
3031 | unload_offset = offset + | |
3032 | (has_delay_slot(op->c) && !op_flag_no_ds(op->flags)); | |
3033 | ||
3034 | lightrec_do_early_unload(state, block, unload_offset); | |
98fa08a5 | 3035 | } |
cb72ea13 PC |
3036 | |
3037 | state->no_load_delay = false; | |
d16005f8 | 3038 | } |