2 * Copyright (C) 2014-2020 Paul Cercueil <paul@crapouillou.net>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
15 #include "blockcache.h"
18 #include "disassembler.h"
20 #include "interpreter.h"
22 #include "memmanager.h"
23 #include "recompiler.h"
25 #include "optimizer.h"
28 #include <lightning.h>
30 #if ENABLE_THREADED_COMPILER
31 #include <stdatomic.h>
40 #define GENMASK(h, l) \
41 (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
43 static struct block * lightrec_precompile_block(struct lightrec_state *state,
46 static void __segfault_cb(struct lightrec_state *state, u32 addr)
48 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
49 pr_err("Segmentation fault in recompiled code: invalid "
50 "load/store at address 0x%08x\n", addr);
53 static u32 lightrec_rw_ops(struct lightrec_state *state, union code op,
54 const struct lightrec_mem_map_ops *ops, u32 addr, u32 data)
58 ops->sb(state, addr, (u8) data);
61 ops->sh(state, addr, (u16) data);
66 ops->sw(state, addr, data);
69 return (s32) (s8) ops->lb(state, addr);
71 return ops->lb(state, addr);
73 return (s32) (s16) ops->lh(state, addr);
75 return ops->lh(state, addr);
78 return ops->lw(state, addr);
82 static void lightrec_invalidate_map(struct lightrec_state *state,
83 const struct lightrec_mem_map *map, u32 addr)
85 if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM])
86 state->code_lut[lut_offset(addr)] = NULL;
89 static const struct lightrec_mem_map *
90 lightrec_get_map(struct lightrec_state *state, u32 kaddr)
94 for (i = 0; i < state->nb_maps; i++) {
95 const struct lightrec_mem_map *map = &state->maps[i];
97 if (kaddr >= map->pc && kaddr < map->pc + map->length)
104 u32 lightrec_rw(struct lightrec_state *state, union code op,
105 u32 addr, u32 data, u16 *flags)
107 const struct lightrec_mem_map *map;
108 u32 shift, mem_data, mask, pc;
112 addr += (s16) op.i.imm;
113 kaddr = kunseg(addr);
115 map = lightrec_get_map(state, kaddr);
117 __segfault_cb(state, addr);
123 if (unlikely(map->ops)) {
125 *flags |= LIGHTREC_HW_IO;
127 return lightrec_rw_ops(state, op, map->ops, addr, data);
130 while (map->mirror_of)
131 map = map->mirror_of;
134 *flags |= LIGHTREC_DIRECT_IO;
137 new_addr = (uintptr_t) map->address + kaddr;
141 *(u8 *) new_addr = (u8) data;
142 if (!state->invalidate_from_dma_only)
143 lightrec_invalidate_map(state, map, kaddr);
146 *(u16 *) new_addr = HTOLE16((u16) data);
147 if (!state->invalidate_from_dma_only)
148 lightrec_invalidate_map(state, map, kaddr);
152 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
153 mask = GENMASK(31, (shift + 1) * 8);
155 *(u32 *)(new_addr & ~3) = HTOLE32((data >> ((3 - shift) * 8))
156 | (mem_data & mask));
157 if (!state->invalidate_from_dma_only)
158 lightrec_invalidate_map(state, map, kaddr & ~0x3);
162 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
163 mask = (1 << (shift * 8)) - 1;
165 *(u32 *)(new_addr & ~3) = HTOLE32((data << (shift * 8))
166 | (mem_data & mask));
167 if (!state->invalidate_from_dma_only)
168 lightrec_invalidate_map(state, map, kaddr & ~0x3);
171 *(u32 *) new_addr = HTOLE32(data);
172 if (!state->invalidate_from_dma_only)
173 lightrec_invalidate_map(state, map, kaddr);
176 *(u32 *) new_addr = HTOLE32(state->ops.cop2_ops.mfc(state,
178 if (!state->invalidate_from_dma_only)
179 lightrec_invalidate_map(state, map, kaddr);
182 return (s32) *(s8 *) new_addr;
184 return *(u8 *) new_addr;
186 return (s32)(s16) LE16TOH(*(u16 *) new_addr);
188 return LE16TOH(*(u16 *) new_addr);
191 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
192 mask = (1 << (24 - shift * 8)) - 1;
194 return (data & mask) | (mem_data << (24 - shift * 8));
197 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
198 mask = GENMASK(31, 32 - shift * 8);
200 return (data & mask) | (mem_data >> (shift * 8));
202 state->ops.cop2_ops.mtc(state, op.i.rt,
203 LE32TOH(*(u32 *) new_addr));
207 return LE32TOH(*(u32 *) new_addr);
211 static void lightrec_rw_helper(struct lightrec_state *state,
212 union code op, u16 *flags)
214 u32 ret = lightrec_rw(state, op,
215 state->native_reg_cache[op.i.rs],
216 state->native_reg_cache[op.i.rt], flags);
227 state->native_reg_cache[op.i.rt] = ret;
228 default: /* fall-through */
233 static void lightrec_rw_cb(struct lightrec_state *state, union code op)
235 lightrec_rw_helper(state, op, NULL);
238 static void lightrec_rw_generic_cb(struct lightrec_state *state,
239 struct opcode *op, struct block *block)
241 bool was_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
243 lightrec_rw_helper(state, op->c, &op->flags);
246 pr_debug("Opcode of block at PC 0x%08x offset 0x%x has been "
247 "tagged - flag for recompilation\n",
248 block->pc, op->offset << 2);
250 lightrec_mark_for_recompilation(state->block_cache, block);
254 u32 lightrec_mfc(struct lightrec_state *state, union code op)
256 bool is_cfc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CFC0) ||
257 (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CFC2);
258 u32 (*func)(struct lightrec_state *, u8);
259 const struct lightrec_cop_ops *ops;
261 if (op.i.op == OP_CP0)
262 ops = &state->ops.cop0_ops;
264 ops = &state->ops.cop2_ops;
271 return (*func)(state, op.r.rd);
274 static void lightrec_mfc_cb(struct lightrec_state *state, union code op)
276 u32 rt = lightrec_mfc(state, op);
279 state->native_reg_cache[op.r.rt] = rt;
282 void lightrec_mtc(struct lightrec_state *state, union code op, u32 data)
284 bool is_ctc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CTC0) ||
285 (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CTC2);
286 void (*func)(struct lightrec_state *, u8, u32);
287 const struct lightrec_cop_ops *ops;
289 if (op.i.op == OP_CP0)
290 ops = &state->ops.cop0_ops;
292 ops = &state->ops.cop2_ops;
299 (*func)(state, op.r.rd, data);
302 static void lightrec_mtc_cb(struct lightrec_state *state, union code op)
304 lightrec_mtc(state, op, state->native_reg_cache[op.r.rt]);
307 static void lightrec_rfe_cb(struct lightrec_state *state, union code op)
311 /* Read CP0 Status register (r12) */
312 status = state->ops.cop0_ops.mfc(state, 12);
314 /* Switch the bits */
315 status = ((status & 0x3c) >> 2) | (status & ~0xf);
318 state->ops.cop0_ops.ctc(state, 12, status);
321 static void lightrec_cp_cb(struct lightrec_state *state, union code op)
323 void (*func)(struct lightrec_state *, u32);
325 if ((op.opcode >> 25) & 1)
326 func = state->ops.cop2_ops.op;
328 func = state->ops.cop0_ops.op;
330 (*func)(state, op.opcode);
333 static void lightrec_syscall_cb(struct lightrec_state *state, union code op)
335 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SYSCALL);
338 static void lightrec_break_cb(struct lightrec_state *state, union code op)
340 lightrec_set_exit_flags(state, LIGHTREC_EXIT_BREAK);
343 struct block * lightrec_get_block(struct lightrec_state *state, u32 pc)
345 struct block *block = lightrec_find_block(state->block_cache, pc);
347 if (block && lightrec_block_is_outdated(block)) {
348 pr_debug("Block at PC 0x%08x is outdated!\n", block->pc);
350 /* Make sure the recompiler isn't processing the block we'll
352 if (ENABLE_THREADED_COMPILER)
353 lightrec_recompiler_remove(state->rec, block);
355 lightrec_unregister_block(state->block_cache, block);
356 lightrec_free_block(block);
361 block = lightrec_precompile_block(state, pc);
363 pr_err("Unable to recompile block at PC 0x%x\n", pc);
364 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
368 lightrec_register_block(state->block_cache, block);
374 static void * get_next_block_func(struct lightrec_state *state, u32 pc)
377 bool should_recompile;
381 func = state->code_lut[lut_offset(pc)];
382 if (func && func != state->get_next_block)
385 block = lightrec_get_block(state, pc);
387 if (unlikely(!block))
390 should_recompile = block->flags & BLOCK_SHOULD_RECOMPILE;
392 if (unlikely(should_recompile)) {
393 pr_debug("Block at PC 0x%08x should recompile"
394 " - freeing old code\n", pc);
396 if (ENABLE_THREADED_COMPILER)
397 lightrec_recompiler_remove(state->rec, block);
399 remove_from_code_lut(state->block_cache, block);
400 lightrec_unregister(MEM_FOR_CODE, block->code_size);
402 _jit_destroy_state(block->_jit);
404 block->function = NULL;
405 block->flags &= ~BLOCK_SHOULD_RECOMPILE;
408 if (ENABLE_THREADED_COMPILER && likely(!should_recompile))
409 func = lightrec_recompiler_run_first_pass(block, &pc);
411 func = block->function;
416 /* Block wasn't compiled yet - run the interpreter */
417 if (!ENABLE_THREADED_COMPILER &&
418 ((ENABLE_FIRST_PASS && likely(!should_recompile)) ||
419 unlikely(block->flags & BLOCK_NEVER_COMPILE)))
420 pc = lightrec_emulate_block(block, pc);
422 if (likely(!(block->flags & BLOCK_NEVER_COMPILE))) {
423 /* Then compile it using the profiled data */
424 if (ENABLE_THREADED_COMPILER)
425 lightrec_recompiler_add(state->rec, block);
427 lightrec_compile_block(block);
430 if (state->exit_flags != LIGHTREC_EXIT_NORMAL ||
431 state->current_cycle >= state->target_cycle) {
438 static s32 c_generic_function_wrapper(struct lightrec_state *state,
440 void (*f)(struct lightrec_state *,
443 struct opcode *op, struct block *block)
445 state->current_cycle = state->target_cycle - cycles_delta;
447 (*f)(state, op, block);
449 return state->target_cycle - state->current_cycle;
452 static s32 c_function_wrapper(struct lightrec_state *state, s32 cycles_delta,
453 void (*f)(struct lightrec_state *, union code),
456 state->current_cycle = state->target_cycle - cycles_delta;
460 return state->target_cycle - state->current_cycle;
463 static struct block * generate_wrapper(struct lightrec_state *state,
464 void *f, bool generic)
470 jit_word_t code_size;
471 jit_node_t *to_tramp, *to_fn_epilog;
473 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
477 _jit = jit_new_state();
481 jit_name("RW wrapper");
482 jit_note(__FILE__, __LINE__);
484 /* Wrapper entry point */
487 stack_ptr = jit_allocai(sizeof(uintptr_t) * NUM_TEMPS);
489 for (i = 0; i < NUM_TEMPS; i++)
490 jit_stxi(stack_ptr + i * sizeof(uintptr_t), JIT_FP, JIT_R(i));
492 /* Jump to the trampoline */
493 to_tramp = jit_jmpi();
495 /* The trampoline will jump back here */
496 to_fn_epilog = jit_label();
498 for (i = 0; i < NUM_TEMPS; i++)
499 jit_ldxi(JIT_R(i), JIT_FP, stack_ptr + i * sizeof(uintptr_t));
504 /* Trampoline entry point.
505 * The sole purpose of the trampoline is to cheese Lightning not to
506 * save/restore the callee-saved register LIGHTREC_REG_CYCLE, since we
507 * do want to return to the caller with this register modified. */
513 jit_pushargr(LIGHTREC_REG_STATE);
514 jit_pushargr(LIGHTREC_REG_CYCLE);
515 jit_pushargi((uintptr_t)f);
516 jit_pushargr(JIT_R0);
518 jit_pushargr(JIT_R1);
519 jit_finishi(c_generic_function_wrapper);
521 jit_finishi(c_function_wrapper);
525 jit_retval_i(LIGHTREC_REG_CYCLE);
527 jit_retval(LIGHTREC_REG_CYCLE);
530 jit_patch_at(jit_jmpi(), to_fn_epilog);
533 block->state = state;
535 block->function = jit_emit();
536 block->opcode_list = NULL;
540 jit_get_code(&code_size);
541 lightrec_register(MEM_FOR_CODE, code_size);
543 block->code_size = code_size;
545 if (ENABLE_DISASSEMBLER) {
546 pr_debug("Wrapper block:\n");
554 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
556 pr_err("Unable to compile wrapper: Out of memory\n");
560 static struct block * generate_dispatcher(struct lightrec_state *state)
564 jit_node_t *to_end, *to_end2, *to_c, *loop, *addr, *addr2;
567 jit_word_t code_size;
569 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
573 _jit = jit_new_state();
577 jit_name("dispatcher");
578 jit_note(__FILE__, __LINE__);
583 jit_getarg(JIT_R0, jit_arg());
585 jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg());
587 jit_getarg(LIGHTREC_REG_CYCLE, jit_arg());
590 /* Force all callee-saved registers to be pushed on the stack */
591 for (i = 0; i < NUM_REGS; i++)
592 jit_movr(JIT_V(i), JIT_V(i));
594 /* Pass lightrec_state structure to blocks, using the last callee-saved
595 * register that Lightning provides */
596 jit_movi(LIGHTREC_REG_STATE, (intptr_t) state);
600 /* Call the block's code */
603 /* The block will jump here, with the number of cycles remaining in
604 * LIGHTREC_REG_CYCLE */
605 addr2 = jit_indirect();
607 /* Jump to end if state->target_cycle < state->current_cycle */
608 to_end = jit_blei(LIGHTREC_REG_CYCLE, 0);
610 /* Convert next PC to KUNSEG and avoid mirrors */
611 ram_len = state->maps[PSX_MAP_KERNEL_USER_RAM].length;
612 jit_andi(JIT_R0, JIT_V0, 0x10000000 | (ram_len - 1));
613 to_c = jit_bgei(JIT_R0, ram_len);
615 /* Fast path: code is running from RAM, use the code LUT */
617 jit_lshi(JIT_R0, JIT_R0, 1);
619 jit_addr(JIT_R0, JIT_R0, LIGHTREC_REG_STATE);
620 jit_ldxi(JIT_R0, JIT_R0, offsetof(struct lightrec_state, code_lut));
622 /* If we get non-NULL, loop */
623 jit_patch_at(jit_bnei(JIT_R0, 0), loop);
625 /* Slow path: call C function get_next_block_func() */
628 if (ENABLE_FIRST_PASS) {
629 /* We may call the interpreter - update state->current_cycle */
630 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
631 offsetof(struct lightrec_state, target_cycle));
632 jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE);
633 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
634 LIGHTREC_REG_STATE, JIT_R1);
637 /* The code LUT will be set to this address when the block at the target
638 * PC has been preprocessed but not yet compiled by the threaded
640 addr = jit_indirect();
642 /* Get the next block */
644 jit_pushargr(LIGHTREC_REG_STATE);
645 jit_pushargr(JIT_V0);
646 jit_finishi(&get_next_block_func);
649 if (ENABLE_FIRST_PASS) {
650 /* The interpreter may have updated state->current_cycle and
651 * state->target_cycle - recalc the delta */
652 jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE,
653 offsetof(struct lightrec_state, current_cycle));
654 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
655 offsetof(struct lightrec_state, target_cycle));
656 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1);
659 /* If we get non-NULL, loop */
660 jit_patch_at(jit_bnei(JIT_R0, 0), loop);
662 to_end2 = jit_jmpi();
664 /* When exiting, the recompiled code will jump to that address */
665 jit_note(__FILE__, __LINE__);
668 /* Store back the next_pc to the lightrec_state structure */
669 offset = offsetof(struct lightrec_state, next_pc);
670 jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0);
674 jit_retr(LIGHTREC_REG_CYCLE);
677 block->state = state;
679 block->function = jit_emit();
680 block->opcode_list = NULL;
684 jit_get_code(&code_size);
685 lightrec_register(MEM_FOR_CODE, code_size);
687 block->code_size = code_size;
689 state->eob_wrapper_func = jit_address(addr2);
690 state->get_next_block = jit_address(addr);
692 if (ENABLE_DISASSEMBLER) {
693 pr_debug("Dispatcher block:\n");
702 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
704 pr_err("Unable to compile dispatcher: Out of memory\n");
708 union code lightrec_read_opcode(struct lightrec_state *state, u32 pc)
710 u32 addr, kunseg_pc = kunseg(pc);
712 const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
714 addr = kunseg_pc - map->pc;
716 while (map->mirror_of)
717 map = map->mirror_of;
719 code = map->address + addr;
721 return (union code) *code;
724 static struct block * lightrec_precompile_block(struct lightrec_state *state,
730 u32 addr, kunseg_pc = kunseg(pc);
731 const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
737 addr = kunseg_pc - map->pc;
739 while (map->mirror_of)
740 map = map->mirror_of;
742 code = map->address + addr;
744 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
746 pr_err("Unable to recompile block: Out of memory\n");
750 list = lightrec_disassemble(state, code, &length);
752 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
757 block->state = state;
759 block->function = NULL;
760 block->opcode_list = list;
764 block->code_size = 0;
765 #if ENABLE_THREADED_COMPILER
766 block->op_list_freed = (atomic_flag)ATOMIC_FLAG_INIT;
768 block->nb_ops = length / sizeof(u32);
770 lightrec_optimize(block);
772 length = block->nb_ops * sizeof(u32);
774 lightrec_register(MEM_FOR_MIPS_CODE, length);
776 if (ENABLE_DISASSEMBLER) {
777 pr_debug("Disassembled block at PC: 0x%x\n", block->pc);
778 lightrec_print_disassembly(block, code, length);
781 pr_debug("Block size: %lu opcodes\n", block->nb_ops);
783 /* If the first opcode is an 'impossible' branch, never compile the
785 if (list->flags & LIGHTREC_EMULATE_BRANCH)
786 block->flags |= BLOCK_NEVER_COMPILE;
788 block->hash = lightrec_calculate_block_hash(block);
793 static bool lightrec_block_is_fully_tagged(struct block *block)
797 for (op = block->opcode_list; op; op = op->next) {
798 /* Verify that all load/stores of the opcode list
799 * Check all loads/stores of the opcode list and mark the
800 * block as fully compiled if they all have been tagged. */
801 switch (op->c.i.op) {
816 if (!(op->flags & (LIGHTREC_DIRECT_IO |
819 default: /* fall-through */
827 int lightrec_compile_block(struct block *block)
829 struct lightrec_state *state = block->state;
830 bool op_list_freed = false, fully_tagged = false;
833 jit_node_t *start_of_block;
834 bool skip_next = false;
835 jit_word_t code_size;
839 fully_tagged = lightrec_block_is_fully_tagged(block);
841 block->flags |= BLOCK_FULLY_TAGGED;
843 _jit = jit_new_state();
849 lightrec_regcache_reset(state->reg_cache);
851 state->nb_branches = 0;
852 state->nb_local_branches = 0;
853 state->nb_targets = 0;
858 start_of_block = jit_label();
860 for (elm = block->opcode_list; elm; elm = elm->next) {
861 next_pc = block->pc + elm->offset * sizeof(u32);
868 state->cycles += lightrec_cycles_of_opcode(elm->c);
870 if (elm->flags & LIGHTREC_EMULATE_BRANCH) {
871 pr_debug("Branch at offset 0x%x will be emulated\n",
873 lightrec_emit_eob(block, elm, next_pc);
874 skip_next = !(elm->flags & LIGHTREC_NO_DS);
875 } else if (elm->opcode) {
876 lightrec_rec_opcode(block, elm, next_pc);
877 skip_next = has_delay_slot(elm->c) &&
878 !(elm->flags & LIGHTREC_NO_DS);
880 /* FIXME: GNU Lightning on Windows seems to use our
881 * mapped registers as temporaries. Until the actual bug
882 * is found and fixed, unconditionally mark our
883 * registers as live here. */
884 lightrec_regcache_mark_live(state->reg_cache, _jit);
889 for (i = 0; i < state->nb_branches; i++)
890 jit_patch(state->branches[i]);
892 for (i = 0; i < state->nb_local_branches; i++) {
893 struct lightrec_branch *branch = &state->local_branches[i];
895 pr_debug("Patch local branch to offset 0x%x\n",
896 branch->target << 2);
898 if (branch->target == 0) {
899 jit_patch_at(branch->branch, start_of_block);
903 for (j = 0; j < state->nb_targets; j++) {
904 if (state->targets[j].offset == branch->target) {
905 jit_patch_at(branch->branch,
906 state->targets[j].label);
911 if (j == state->nb_targets)
912 pr_err("Unable to find branch target\n");
915 jit_ldxi(JIT_R0, LIGHTREC_REG_STATE,
916 offsetof(struct lightrec_state, eob_wrapper_func));
923 block->function = jit_emit();
925 /* Add compiled function to the LUT */
926 state->code_lut[lut_offset(block->pc)] = block->function;
928 jit_get_code(&code_size);
929 lightrec_register(MEM_FOR_CODE, code_size);
931 block->code_size = code_size;
933 if (ENABLE_DISASSEMBLER) {
934 pr_debug("Compiling block at PC: 0x%x\n", block->pc);
940 #if ENABLE_THREADED_COMPILER
942 op_list_freed = atomic_flag_test_and_set(&block->op_list_freed);
944 if (fully_tagged && !op_list_freed) {
945 pr_debug("Block PC 0x%08x is fully tagged"
946 " - free opcode list\n", block->pc);
947 lightrec_free_opcode_list(state, block->opcode_list);
948 block->opcode_list = NULL;
954 u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle)
956 s32 (*func)(void *, s32) = (void *)state->dispatcher->function;
960 state->exit_flags = LIGHTREC_EXIT_NORMAL;
962 /* Handle the cycle counter overflowing */
963 if (unlikely(target_cycle < state->current_cycle))
964 target_cycle = UINT_MAX;
966 state->target_cycle = target_cycle;
968 block_trace = get_next_block_func(state, pc);
970 cycles_delta = state->target_cycle - state->current_cycle;
972 cycles_delta = (*func)(block_trace, cycles_delta);
974 state->current_cycle = state->target_cycle - cycles_delta;
977 return state->next_pc;
980 u32 lightrec_execute_one(struct lightrec_state *state, u32 pc)
982 return lightrec_execute(state, pc, state->current_cycle);
985 u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc)
987 struct block *block = lightrec_get_block(state, pc);
991 state->exit_flags = LIGHTREC_EXIT_NORMAL;
993 return lightrec_emulate_block(block, pc);
996 void lightrec_free_block(struct block *block)
998 lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32));
999 if (block->opcode_list)
1000 lightrec_free_opcode_list(block->state, block->opcode_list);
1002 _jit_destroy_state(block->_jit);
1003 lightrec_unregister(MEM_FOR_CODE, block->code_size);
1004 lightrec_free(block->state, MEM_FOR_IR, sizeof(*block), block);
1007 struct lightrec_state * lightrec_init(char *argv0,
1008 const struct lightrec_mem_map *map,
1010 const struct lightrec_ops *ops)
1012 struct lightrec_state *state;
1014 /* Sanity-check ops */
1016 !ops->cop0_ops.mfc || !ops->cop0_ops.cfc || !ops->cop0_ops.mtc ||
1017 !ops->cop0_ops.ctc || !ops->cop0_ops.op ||
1018 !ops->cop2_ops.mfc || !ops->cop2_ops.cfc || !ops->cop2_ops.mtc ||
1019 !ops->cop2_ops.ctc || !ops->cop2_ops.op) {
1020 pr_err("Missing callbacks in lightrec_ops structure\n");
1026 state = calloc(1, sizeof(*state) +
1027 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1029 goto err_finish_jit;
1031 lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) +
1032 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1035 state->tinymm = tinymm_init(malloc, free, 4096);
1037 goto err_free_state;
1040 state->block_cache = lightrec_blockcache_init(state);
1041 if (!state->block_cache)
1042 goto err_free_tinymm;
1044 state->reg_cache = lightrec_regcache_init(state);
1045 if (!state->reg_cache)
1046 goto err_free_block_cache;
1048 if (ENABLE_THREADED_COMPILER) {
1049 state->rec = lightrec_recompiler_init(state);
1051 goto err_free_reg_cache;
1054 state->nb_maps = nb;
1057 memcpy(&state->ops, ops, sizeof(*ops));
1059 state->dispatcher = generate_dispatcher(state);
1060 if (!state->dispatcher)
1061 goto err_free_recompiler;
1063 state->rw_generic_wrapper = generate_wrapper(state,
1064 lightrec_rw_generic_cb,
1066 if (!state->rw_generic_wrapper)
1067 goto err_free_dispatcher;
1069 state->rw_wrapper = generate_wrapper(state, lightrec_rw_cb, false);
1070 if (!state->rw_wrapper)
1071 goto err_free_generic_rw_wrapper;
1073 state->mfc_wrapper = generate_wrapper(state, lightrec_mfc_cb, false);
1074 if (!state->mfc_wrapper)
1075 goto err_free_rw_wrapper;
1077 state->mtc_wrapper = generate_wrapper(state, lightrec_mtc_cb, false);
1078 if (!state->mtc_wrapper)
1079 goto err_free_mfc_wrapper;
1081 state->rfe_wrapper = generate_wrapper(state, lightrec_rfe_cb, false);
1082 if (!state->rfe_wrapper)
1083 goto err_free_mtc_wrapper;
1085 state->cp_wrapper = generate_wrapper(state, lightrec_cp_cb, false);
1086 if (!state->cp_wrapper)
1087 goto err_free_rfe_wrapper;
1089 state->syscall_wrapper = generate_wrapper(state, lightrec_syscall_cb,
1091 if (!state->syscall_wrapper)
1092 goto err_free_cp_wrapper;
1094 state->break_wrapper = generate_wrapper(state, lightrec_break_cb,
1096 if (!state->break_wrapper)
1097 goto err_free_syscall_wrapper;
1099 state->rw_generic_func = state->rw_generic_wrapper->function;
1100 state->rw_func = state->rw_wrapper->function;
1101 state->mfc_func = state->mfc_wrapper->function;
1102 state->mtc_func = state->mtc_wrapper->function;
1103 state->rfe_func = state->rfe_wrapper->function;
1104 state->cp_func = state->cp_wrapper->function;
1105 state->syscall_func = state->syscall_wrapper->function;
1106 state->break_func = state->break_wrapper->function;
1108 map = &state->maps[PSX_MAP_BIOS];
1109 state->offset_bios = (uintptr_t)map->address - map->pc;
1111 map = &state->maps[PSX_MAP_SCRATCH_PAD];
1112 state->offset_scratch = (uintptr_t)map->address - map->pc;
1114 map = &state->maps[PSX_MAP_KERNEL_USER_RAM];
1115 state->offset_ram = (uintptr_t)map->address - map->pc;
1117 if (state->maps[PSX_MAP_MIRROR1].address == map->address + 0x200000 &&
1118 state->maps[PSX_MAP_MIRROR2].address == map->address + 0x400000 &&
1119 state->maps[PSX_MAP_MIRROR3].address == map->address + 0x600000)
1120 state->mirrors_mapped = true;
1124 err_free_syscall_wrapper:
1125 lightrec_free_block(state->syscall_wrapper);
1126 err_free_cp_wrapper:
1127 lightrec_free_block(state->cp_wrapper);
1128 err_free_rfe_wrapper:
1129 lightrec_free_block(state->rfe_wrapper);
1130 err_free_mtc_wrapper:
1131 lightrec_free_block(state->mtc_wrapper);
1132 err_free_mfc_wrapper:
1133 lightrec_free_block(state->mfc_wrapper);
1134 err_free_rw_wrapper:
1135 lightrec_free_block(state->rw_wrapper);
1136 err_free_generic_rw_wrapper:
1137 lightrec_free_block(state->rw_generic_wrapper);
1138 err_free_dispatcher:
1139 lightrec_free_block(state->dispatcher);
1140 err_free_recompiler:
1141 if (ENABLE_THREADED_COMPILER)
1142 lightrec_free_recompiler(state->rec);
1144 lightrec_free_regcache(state->reg_cache);
1145 err_free_block_cache:
1146 lightrec_free_block_cache(state->block_cache);
1149 tinymm_shutdown(state->tinymm);
1152 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1153 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1160 void lightrec_destroy(struct lightrec_state *state)
1162 if (ENABLE_THREADED_COMPILER)
1163 lightrec_free_recompiler(state->rec);
1165 lightrec_free_regcache(state->reg_cache);
1166 lightrec_free_block_cache(state->block_cache);
1167 lightrec_free_block(state->dispatcher);
1168 lightrec_free_block(state->rw_generic_wrapper);
1169 lightrec_free_block(state->rw_wrapper);
1170 lightrec_free_block(state->mfc_wrapper);
1171 lightrec_free_block(state->mtc_wrapper);
1172 lightrec_free_block(state->rfe_wrapper);
1173 lightrec_free_block(state->cp_wrapper);
1174 lightrec_free_block(state->syscall_wrapper);
1175 lightrec_free_block(state->break_wrapper);
1179 tinymm_shutdown(state->tinymm);
1181 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1182 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1186 void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len)
1188 u32 kaddr = kunseg(addr & ~0x3);
1189 const struct lightrec_mem_map *map = lightrec_get_map(state, kaddr);
1192 while (map->mirror_of)
1193 map = map->mirror_of;
1195 if (map != &state->maps[PSX_MAP_KERNEL_USER_RAM])
1198 /* Handle mirrors */
1199 kaddr &= (state->maps[PSX_MAP_KERNEL_USER_RAM].length - 1);
1201 for (; len > 4; len -= 4, kaddr += 4)
1202 lightrec_invalidate_map(state, map, kaddr);
1204 lightrec_invalidate_map(state, map, kaddr);
1208 void lightrec_invalidate_all(struct lightrec_state *state)
1210 memset(state->code_lut, 0, sizeof(*state->code_lut) * CODE_LUT_SIZE);
1213 void lightrec_set_invalidate_mode(struct lightrec_state *state, bool dma_only)
1215 if (state->invalidate_from_dma_only != dma_only)
1216 lightrec_invalidate_all(state);
1218 state->invalidate_from_dma_only = dma_only;
1221 void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags)
1223 if (flags != LIGHTREC_EXIT_NORMAL) {
1224 state->exit_flags |= flags;
1225 state->target_cycle = state->current_cycle;
1229 u32 lightrec_exit_flags(struct lightrec_state *state)
1231 return state->exit_flags;
1234 void lightrec_dump_registers(struct lightrec_state *state, u32 regs[34])
1236 memcpy(regs, state->native_reg_cache, sizeof(state->native_reg_cache));
1239 void lightrec_restore_registers(struct lightrec_state *state, u32 regs[34])
1241 memcpy(state->native_reg_cache, regs, sizeof(state->native_reg_cache));
1244 u32 lightrec_current_cycle_count(const struct lightrec_state *state)
1246 return state->current_cycle;
1249 void lightrec_reset_cycle_count(struct lightrec_state *state, u32 cycles)
1251 state->current_cycle = cycles;
1253 if (state->target_cycle < cycles)
1254 state->target_cycle = cycles;
1257 void lightrec_set_target_cycle_count(struct lightrec_state *state, u32 cycles)
1259 if (state->exit_flags == LIGHTREC_EXIT_NORMAL) {
1260 if (cycles < state->current_cycle)
1261 cycles = state->current_cycle;
1263 state->target_cycle = cycles;