notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #417 from negativeExponent/libretro
[pcsx_rearmed.git] / deps / lightrec / lightrec.c
CommitLineData
d16005f8
PC		 1/*
2 * Copyright (C) 2014-2020 Paul Cercueil <paul@crapouillou.net>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 */
14
15#include "blockcache.h"
16#include "config.h"
17#include "debug.h"
18#include "disassembler.h"
19#include "emitter.h"
20#include "interpreter.h"
21#include "lightrec.h"
22#include "memmanager.h"
23#include "recompiler.h"
24#include "regcache.h"
25#include "optimizer.h"
26
27#include <errno.h>
28#include <lightning.h>
29#include <limits.h>
30#if ENABLE_THREADED_COMPILER
31#include <stdatomic.h>
32#endif
33#include <stdbool.h>
34#include <stddef.h>
35#include <string.h>
36#if ENABLE_TINYMM
37#include <tinymm.h>
38#endif
39
40#define GENMASK(h, l) \
41 (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
42
43static struct block * lightrec_precompile_block(struct lightrec_state *state,
44 u32 pc);
45
46static void __segfault_cb(struct lightrec_state *state, u32 addr)
47{
48 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
49 pr_err("Segmentation fault in recompiled code: invalid "
50 "load/store at address 0x%08x\n", addr);
51}
52
53static u32 lightrec_rw_ops(struct lightrec_state *state, union code op,
54 const struct lightrec_mem_map_ops *ops, u32 addr, u32 data)
55{
56 switch (op.i.op) {
57 case OP_SB:
58 ops->sb(state, addr, (u8) data);
59 return 0;
60 case OP_SH:
61 ops->sh(state, addr, (u16) data);
62 return 0;
63 case OP_SWL:
64 case OP_SWR:
65 case OP_SW:
66 ops->sw(state, addr, data);
67 return 0;
68 case OP_LB:
69 return (s32) (s8) ops->lb(state, addr);
70 case OP_LBU:
71 return ops->lb(state, addr);
72 case OP_LH:
73 return (s32) (s16) ops->lh(state, addr);
74 case OP_LHU:
75 return ops->lh(state, addr);
76 case OP_LW:
77 default:
78 return ops->lw(state, addr);
79 }
80}
81
82static void lightrec_invalidate_map(struct lightrec_state *state,
83 const struct lightrec_mem_map *map, u32 addr)
84{
85 if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM])
86 state->code_lut[lut_offset(addr)] = NULL;
87}
88
89static const struct lightrec_mem_map *
90lightrec_get_map(struct lightrec_state *state, u32 kaddr)
91{
92 unsigned int i;
93
94 for (i = 0; i < state->nb_maps; i++) {
95 const struct lightrec_mem_map *map = &state->maps[i];
96
97 if (kaddr >= map->pc && kaddr < map->pc + map->length)
98 return map;
99 }
100
101 return NULL;
102}
103
104u32 lightrec_rw(struct lightrec_state *state, union code op,
105 u32 addr, u32 data, u16 *flags)
106{
107 const struct lightrec_mem_map *map;
108 u32 shift, mem_data, mask, pc;
109 uintptr_t new_addr;
110 u32 kaddr;
111
112 addr += (s16) op.i.imm;
113 kaddr = kunseg(addr);
114
115 map = lightrec_get_map(state, kaddr);
116 if (!map) {
117 __segfault_cb(state, addr);
118 return 0;
119 }
120
121 pc = map->pc;
122
123 if (unlikely(map->ops)) {
124 if (flags)
125 *flags |= LIGHTREC_HW_IO;
126
127 return lightrec_rw_ops(state, op, map->ops, addr, data);
128 }
129
130 while (map->mirror_of)
131 map = map->mirror_of;
132
133 if (flags)
134 *flags |= LIGHTREC_DIRECT_IO;
135
136 kaddr -= pc;
137 new_addr = (uintptr_t) map->address + kaddr;
138
139 switch (op.i.op) {
140 case OP_SB:
141 *(u8 *) new_addr = (u8) data;
142 if (!state->invalidate_from_dma_only)
143 lightrec_invalidate_map(state, map, kaddr);
144 return 0;
145 case OP_SH:
146 *(u16 *) new_addr = HTOLE16((u16) data);
147 if (!state->invalidate_from_dma_only)
148 lightrec_invalidate_map(state, map, kaddr);
149 return 0;
150 case OP_SWL:
151 shift = kaddr & 3;
152 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
153 mask = GENMASK(31, (shift + 1) * 8);
154
155 *(u32 *)(new_addr & ~3) = HTOLE32((data >> ((3 - shift) * 8))
156 | (mem_data & mask));
157 if (!state->invalidate_from_dma_only)
158 lightrec_invalidate_map(state, map, kaddr & ~0x3);
159 return 0;
160 case OP_SWR:
161 shift = kaddr & 3;
162 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
163 mask = (1 << (shift * 8)) - 1;
164
165 *(u32 *)(new_addr & ~3) = HTOLE32((data << (shift * 8))
166 | (mem_data & mask));
167 if (!state->invalidate_from_dma_only)
168 lightrec_invalidate_map(state, map, kaddr & ~0x3);
169 return 0;
170 case OP_SW:
171 *(u32 *) new_addr = HTOLE32(data);
172 if (!state->invalidate_from_dma_only)
173 lightrec_invalidate_map(state, map, kaddr);
174 return 0;
175 case OP_SWC2:
176 *(u32 *) new_addr = HTOLE32(state->ops.cop2_ops.mfc(state,
177 op.i.rt));
178 if (!state->invalidate_from_dma_only)
179 lightrec_invalidate_map(state, map, kaddr);
180 return 0;
181 case OP_LB:
182 return (s32) *(s8 *) new_addr;
183 case OP_LBU:
184 return *(u8 *) new_addr;
185 case OP_LH:
186 return (s32)(s16) LE16TOH(*(u16 *) new_addr);
187 case OP_LHU:
188 return LE16TOH(*(u16 *) new_addr);
189 case OP_LWL:
190 shift = kaddr & 3;
191 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
192 mask = (1 << (24 - shift * 8)) - 1;
193
194 return (data & mask) | (mem_data << (24 - shift * 8));
195 case OP_LWR:
196 shift = kaddr & 3;
197 mem_data = LE32TOH(*(u32 *)(new_addr & ~3));
198 mask = GENMASK(31, 32 - shift * 8);
199
200 return (data & mask) | (mem_data >> (shift * 8));
201 case OP_LWC2:
202 state->ops.cop2_ops.mtc(state, op.i.rt,
203 LE32TOH(*(u32 *) new_addr));
204 return 0;
205 case OP_LW:
206 default:
207 return LE32TOH(*(u32 *) new_addr);
208 }
209}
210
211static void lightrec_rw_helper(struct lightrec_state *state,
212 union code op, u16 *flags)
213{
214 u32 ret = lightrec_rw(state, op,
215 state->native_reg_cache[op.i.rs],
216 state->native_reg_cache[op.i.rt], flags);
217
218 switch (op.i.op) {
219 case OP_LB:
220 case OP_LBU:
221 case OP_LH:
222 case OP_LHU:
223 case OP_LWL:
224 case OP_LWR:
225 case OP_LW:
226 if (op.i.rt)
227 state->native_reg_cache[op.i.rt] = ret;
228 default: /* fall-through */
229 break;
230 }
231}
232
233static void lightrec_rw_cb(struct lightrec_state *state, union code op)
234{
235 lightrec_rw_helper(state, op, NULL);
236}
237
238static void lightrec_rw_generic_cb(struct lightrec_state *state,
239 struct opcode *op, struct block *block)
240{
241 bool was_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
242
243 lightrec_rw_helper(state, op->c, &op->flags);
244
245 if (!was_tagged) {
246 pr_debug("Opcode of block at PC 0x%08x offset 0x%x has been "
247 "tagged - flag for recompilation\n",
248 block->pc, op->offset << 2);
249
250 lightrec_mark_for_recompilation(state->block_cache, block);
251 }
252}
253
254u32 lightrec_mfc(struct lightrec_state *state, union code op)
255{
256 bool is_cfc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CFC0) ||
257 (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CFC2);
258 u32 (*func)(struct lightrec_state *, u8);
259 const struct lightrec_cop_ops *ops;
260
261 if (op.i.op == OP_CP0)
262 ops = &state->ops.cop0_ops;
263 else
264 ops = &state->ops.cop2_ops;
265
266 if (is_cfc)
267 func = ops->cfc;
268 else
269 func = ops->mfc;
270
271 return (*func)(state, op.r.rd);
272}
273
274static void lightrec_mfc_cb(struct lightrec_state *state, union code op)
275{
276 u32 rt = lightrec_mfc(state, op);
277
278 if (op.r.rt)
279 state->native_reg_cache[op.r.rt] = rt;
280}
281
282void lightrec_mtc(struct lightrec_state *state, union code op, u32 data)
283{
284 bool is_ctc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CTC0) ||
285 (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CTC2);
286 void (*func)(struct lightrec_state *, u8, u32);
287 const struct lightrec_cop_ops *ops;
288
289 if (op.i.op == OP_CP0)
290 ops = &state->ops.cop0_ops;
291 else
292 ops = &state->ops.cop2_ops;
293
294 if (is_ctc)
295 func = ops->ctc;
296 else
297 func = ops->mtc;
298
299 (*func)(state, op.r.rd, data);
300}
301
302static void lightrec_mtc_cb(struct lightrec_state *state, union code op)
303{
304 lightrec_mtc(state, op, state->native_reg_cache[op.r.rt]);
305}
306
307static void lightrec_rfe_cb(struct lightrec_state *state, union code op)
308{
309 u32 status;
310
311 /* Read CP0 Status register (r12) */
312 status = state->ops.cop0_ops.mfc(state, 12);
313
314 /* Switch the bits */
315 status = ((status & 0x3c) >> 2) | (status & ~0xf);
316
317 /* Write it back */
318 state->ops.cop0_ops.ctc(state, 12, status);
319}
320
321static void lightrec_cp_cb(struct lightrec_state *state, union code op)
322{
323 void (*func)(struct lightrec_state *, u32);
324
325 if ((op.opcode >> 25) & 1)
326 func = state->ops.cop2_ops.op;
327 else
328 func = state->ops.cop0_ops.op;
329
330 (*func)(state, op.opcode);
331}
332
333static void lightrec_syscall_cb(struct lightrec_state *state, union code op)
334{
335 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SYSCALL);
336}
337
338static void lightrec_break_cb(struct lightrec_state *state, union code op)
339{
340 lightrec_set_exit_flags(state, LIGHTREC_EXIT_BREAK);
341}
342
343struct block * lightrec_get_block(struct lightrec_state *state, u32 pc)
344{
345 struct block *block = lightrec_find_block(state->block_cache, pc);
346
347 if (block && lightrec_block_is_outdated(block)) {
348 pr_debug("Block at PC 0x%08x is outdated!\n", block->pc);
349
350 /* Make sure the recompiler isn't processing the block we'll
351 * destroy */
352 if (ENABLE_THREADED_COMPILER)
353 lightrec_recompiler_remove(state->rec, block);
354
355 lightrec_unregister_block(state->block_cache, block);
356 lightrec_free_block(block);
357 block = NULL;
358 }
359
360 if (!block) {
361 block = lightrec_precompile_block(state, pc);
362 if (!block) {
363 pr_err("Unable to recompile block at PC 0x%x\n", pc);
364 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
365 return NULL;
366 }
367
368 lightrec_register_block(state->block_cache, block);
369 }
370
371 return block;
372}
373
374static void * get_next_block_func(struct lightrec_state *state, u32 pc)
375{
376 struct block *block;
377 bool should_recompile;
378 void *func;
379
380 for (;;) {
381 func = state->code_lut[lut_offset(pc)];
382 if (func && func != state->get_next_block)
383 return func;
384
385 block = lightrec_get_block(state, pc);
386
387 if (unlikely(!block))
388 return NULL;
389
390 should_recompile = block->flags & BLOCK_SHOULD_RECOMPILE;
391
392 if (unlikely(should_recompile)) {
393 pr_debug("Block at PC 0x%08x should recompile"
394 " - freeing old code\n", pc);
395
396 if (ENABLE_THREADED_COMPILER)
397 lightrec_recompiler_remove(state->rec, block);
398
399 remove_from_code_lut(state->block_cache, block);
400 lightrec_unregister(MEM_FOR_CODE, block->code_size);
401 if (block->_jit)
402 _jit_destroy_state(block->_jit);
403 block->_jit = NULL;
404 block->function = NULL;
405 block->flags &= ~BLOCK_SHOULD_RECOMPILE;
406 }
407
408 if (ENABLE_THREADED_COMPILER && likely(!should_recompile))
409 func = lightrec_recompiler_run_first_pass(block, &pc);
410 else
411 func = block->function;
412
413 if (likely(func))
414 return func;
415
416 /* Block wasn't compiled yet - run the interpreter */
417 if (!ENABLE_THREADED_COMPILER &&
418 ((ENABLE_FIRST_PASS && likely(!should_recompile)) ||
419 unlikely(block->flags & BLOCK_NEVER_COMPILE)))
420 pc = lightrec_emulate_block(block, pc);
421
422 if (likely(!(block->flags & BLOCK_NEVER_COMPILE))) {
423 /* Then compile it using the profiled data */
424 if (ENABLE_THREADED_COMPILER)
425 lightrec_recompiler_add(state->rec, block);
426 else
427 lightrec_compile_block(block);
428 }
429
430 if (state->exit_flags != LIGHTREC_EXIT_NORMAL ||
431 state->current_cycle >= state->target_cycle) {
432 state->next_pc = pc;
433 return NULL;
434 }
435 }
436}
437
438static s32 c_generic_function_wrapper(struct lightrec_state *state,
439 s32 cycles_delta,
440 void (*f)(struct lightrec_state *,
441 struct opcode *,
442 struct block *),
443 struct opcode *op, struct block *block)
444{
445 state->current_cycle = state->target_cycle - cycles_delta;
446
447 (*f)(state, op, block);
448
449 return state->target_cycle - state->current_cycle;
450}
451
452static s32 c_function_wrapper(struct lightrec_state *state, s32 cycles_delta,
453 void (*f)(struct lightrec_state *, union code),
454 union code op)
455{
456 state->current_cycle = state->target_cycle - cycles_delta;
457
458 (*f)(state, op);
459
460 return state->target_cycle - state->current_cycle;
461}
462
463static struct block * generate_wrapper(struct lightrec_state *state,
464 void *f, bool generic)
465{
466 struct block *block;
467 jit_state_t *_jit;
468 unsigned int i;
469 int stack_ptr;
470 jit_word_t code_size;
471 jit_node_t *to_tramp, *to_fn_epilog;
472
473 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
474 if (!block)
475 goto err_no_mem;
476
477 _jit = jit_new_state();
478 if (!_jit)
479 goto err_free_block;
480
481 jit_name("RW wrapper");
482 jit_note(__FILE__, __LINE__);
483
484 /* Wrapper entry point */
485 jit_prolog();
486
487 stack_ptr = jit_allocai(sizeof(uintptr_t) * NUM_TEMPS);
488
489 for (i = 0; i < NUM_TEMPS; i++)
490 jit_stxi(stack_ptr + i * sizeof(uintptr_t), JIT_FP, JIT_R(i));
491
492 /* Jump to the trampoline */
493 to_tramp = jit_jmpi();
494
495 /* The trampoline will jump back here */
496 to_fn_epilog = jit_label();
497
498 for (i = 0; i < NUM_TEMPS; i++)
499 jit_ldxi(JIT_R(i), JIT_FP, stack_ptr + i * sizeof(uintptr_t));
500
501 jit_ret();
502 jit_epilog();
503
504 /* Trampoline entry point.
505 * The sole purpose of the trampoline is to cheese Lightning not to
506 * save/restore the callee-saved register LIGHTREC_REG_CYCLE, since we
507 * do want to return to the caller with this register modified. */
508 jit_prolog();
509 jit_tramp(256);
510 jit_patch(to_tramp);
511
512 jit_prepare();
513 jit_pushargr(LIGHTREC_REG_STATE);
514 jit_pushargr(LIGHTREC_REG_CYCLE);
515 jit_pushargi((uintptr_t)f);
516 jit_pushargr(JIT_R0);
517 if (generic) {
518 jit_pushargr(JIT_R1);
519 jit_finishi(c_generic_function_wrapper);
520 } else {
521 jit_finishi(c_function_wrapper);
522 }
523
524#if __WORDSIZE == 64
525 jit_retval_i(LIGHTREC_REG_CYCLE);
526#else
527 jit_retval(LIGHTREC_REG_CYCLE);
528#endif
529
530 jit_patch_at(jit_jmpi(), to_fn_epilog);
531 jit_epilog();
532
533 block->state = state;
534 block->_jit = _jit;
535 block->function = jit_emit();
536 block->opcode_list = NULL;
537 block->flags = 0;
538 block->nb_ops = 0;
539
540 jit_get_code(&code_size);
541 lightrec_register(MEM_FOR_CODE, code_size);
542
543 block->code_size = code_size;
544
545 if (ENABLE_DISASSEMBLER) {
546 pr_debug("Wrapper block:\n");
547 jit_disassemble();
548 }
549
550 jit_clear_state();
551 return block;
552
553err_free_block:
554 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
555err_no_mem:
556 pr_err("Unable to compile wrapper: Out of memory\n");
557 return NULL;
558}
559
560static struct block * generate_dispatcher(struct lightrec_state *state)
561{
562 struct block *block;
563 jit_state_t *_jit;
564 jit_node_t *to_end, *to_end2, *to_c, *loop, *addr, *addr2;
565 unsigned int i;
566 u32 offset, ram_len;
567 jit_word_t code_size;
568
569 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
570 if (!block)
571 goto err_no_mem;
572
573 _jit = jit_new_state();
574 if (!_jit)
575 goto err_free_block;
576
577 jit_name("dispatcher");
578 jit_note(__FILE__, __LINE__);
579
580 jit_prolog();
581 jit_frame(256);
582
583 jit_getarg(JIT_R0, jit_arg());
584#if __WORDSIZE == 64
585 jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg());
586#else
587 jit_getarg(LIGHTREC_REG_CYCLE, jit_arg());
588#endif
589
590 /* Force all callee-saved registers to be pushed on the stack */
591 for (i = 0; i < NUM_REGS; i++)
592 jit_movr(JIT_V(i), JIT_V(i));
593
594 /* Pass lightrec_state structure to blocks, using the last callee-saved
595 * register that Lightning provides */
596 jit_movi(LIGHTREC_REG_STATE, (intptr_t) state);
597
598 loop = jit_label();
599
600 /* Call the block's code */
601 jit_jmpr(JIT_R0);
602
603 /* The block will jump here, with the number of cycles remaining in
604 * LIGHTREC_REG_CYCLE */
605 addr2 = jit_indirect();
606
607 /* Jump to end if state->target_cycle < state->current_cycle */
608 to_end = jit_blei(LIGHTREC_REG_CYCLE, 0);
609
610 /* Convert next PC to KUNSEG and avoid mirrors */
611 ram_len = state->maps[PSX_MAP_KERNEL_USER_RAM].length;
612 jit_andi(JIT_R0, JIT_V0, 0x10000000 | (ram_len - 1));
613 to_c = jit_bgei(JIT_R0, ram_len);
614
615 /* Fast path: code is running from RAM, use the code LUT */
616#if __WORDSIZE == 64
617 jit_lshi(JIT_R0, JIT_R0, 1);
618#endif
619 jit_addr(JIT_R0, JIT_R0, LIGHTREC_REG_STATE);
620 jit_ldxi(JIT_R0, JIT_R0, offsetof(struct lightrec_state, code_lut));
621
622 /* If we get non-NULL, loop */
623 jit_patch_at(jit_bnei(JIT_R0, 0), loop);
624
625 /* Slow path: call C function get_next_block_func() */
626 jit_patch(to_c);
627
628 if (ENABLE_FIRST_PASS) {
629 /* We may call the interpreter - update state->current_cycle */
630 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
631 offsetof(struct lightrec_state, target_cycle));
632 jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE);
633 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
634 LIGHTREC_REG_STATE, JIT_R1);
635 }
636
637 /* The code LUT will be set to this address when the block at the target
638 * PC has been preprocessed but not yet compiled by the threaded
639 * recompiler */
640 addr = jit_indirect();
641
642 /* Get the next block */
643 jit_prepare();
644 jit_pushargr(LIGHTREC_REG_STATE);
645 jit_pushargr(JIT_V0);
646 jit_finishi(&get_next_block_func);
647 jit_retval(JIT_R0);
648
649 if (ENABLE_FIRST_PASS) {
650 /* The interpreter may have updated state->current_cycle and
651 * state->target_cycle - recalc the delta */
652 jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE,
653 offsetof(struct lightrec_state, current_cycle));
654 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
655 offsetof(struct lightrec_state, target_cycle));
656 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1);
657 }
658
659 /* If we get non-NULL, loop */
660 jit_patch_at(jit_bnei(JIT_R0, 0), loop);
661
662 to_end2 = jit_jmpi();
663
664 /* When exiting, the recompiled code will jump to that address */
665 jit_note(__FILE__, __LINE__);
666 jit_patch(to_end);
667
668 /* Store back the next_pc to the lightrec_state structure */
669 offset = offsetof(struct lightrec_state, next_pc);
670 jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0);
671
672 jit_patch(to_end2);
673
674 jit_retr(LIGHTREC_REG_CYCLE);
675 jit_epilog();
676
677 block->state = state;
678 block->_jit = _jit;
679 block->function = jit_emit();
680 block->opcode_list = NULL;
681 block->flags = 0;
682 block->nb_ops = 0;
683
684 jit_get_code(&code_size);
685 lightrec_register(MEM_FOR_CODE, code_size);
686
687 block->code_size = code_size;
688
689 state->eob_wrapper_func = jit_address(addr2);
690 state->get_next_block = jit_address(addr);
691
692 if (ENABLE_DISASSEMBLER) {
693 pr_debug("Dispatcher block:\n");
694 jit_disassemble();
695 }
696
697 /* We're done! */
698 jit_clear_state();
699 return block;
700
701err_free_block:
702 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
703err_no_mem:
704 pr_err("Unable to compile dispatcher: Out of memory\n");
705 return NULL;
706}
707
708union code lightrec_read_opcode(struct lightrec_state *state, u32 pc)
709{
710 u32 addr, kunseg_pc = kunseg(pc);
711 const u32 *code;
712 const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
713
714 addr = kunseg_pc - map->pc;
715
716 while (map->mirror_of)
717 map = map->mirror_of;
718
719 code = map->address + addr;
720
721 return (union code) *code;
722}
723
724static struct block * lightrec_precompile_block(struct lightrec_state *state,
725 u32 pc)
726{
727 struct opcode *list;
728 struct block *block;
729 const u32 *code;
730 u32 addr, kunseg_pc = kunseg(pc);
731 const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
732 unsigned int length;
733
734 if (!map)
735 return NULL;
736
737 addr = kunseg_pc - map->pc;
738
739 while (map->mirror_of)
740 map = map->mirror_of;
741
742 code = map->address + addr;
743
744 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
745 if (!block) {
746 pr_err("Unable to recompile block: Out of memory\n");
747 return NULL;
748 }
749
750 list = lightrec_disassemble(state, code, &length);
751 if (!list) {
752 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
753 return NULL;
754 }
755
756 block->pc = pc;
757 block->state = state;
758 block->_jit = NULL;
759 block->function = NULL;
760 block->opcode_list = list;
761 block->map = map;
762 block->next = NULL;
763 block->flags = 0;
764 block->code_size = 0;
765#if ENABLE_THREADED_COMPILER
766 block->op_list_freed = (atomic_flag)ATOMIC_FLAG_INIT;
767#endif
768 block->nb_ops = length / sizeof(u32);
769
770 lightrec_optimize(block);
771
772 length = block->nb_ops * sizeof(u32);
773
774 lightrec_register(MEM_FOR_MIPS_CODE, length);
775
776 if (ENABLE_DISASSEMBLER) {
777 pr_debug("Disassembled block at PC: 0x%x\n", block->pc);
778 lightrec_print_disassembly(block, code, length);
779 }
780
781 pr_debug("Block size: %lu opcodes\n", block->nb_ops);
782
783 /* If the first opcode is an 'impossible' branch, never compile the
784 * block */
785 if (list->flags & LIGHTREC_EMULATE_BRANCH)
786 block->flags |= BLOCK_NEVER_COMPILE;
787
788 block->hash = lightrec_calculate_block_hash(block);
789
790 return block;
791}
792
793static bool lightrec_block_is_fully_tagged(struct block *block)
794{
795 struct opcode *op;
796
797 for (op = block->opcode_list; op; op = op->next) {
798 /* Verify that all load/stores of the opcode list
799 * Check all loads/stores of the opcode list and mark the
800 * block as fully compiled if they all have been tagged. */
801 switch (op->c.i.op) {
802 case OP_LB:
803 case OP_LH:
804 case OP_LWL:
805 case OP_LW:
806 case OP_LBU:
807 case OP_LHU:
808 case OP_LWR:
809 case OP_SB:
810 case OP_SH:
811 case OP_SWL:
812 case OP_SW:
813 case OP_SWR:
814 case OP_LWC2:
815 case OP_SWC2:
816 if (!(op->flags & (LIGHTREC_DIRECT_IO |
817 LIGHTREC_HW_IO)))
818 return false;
819 default: /* fall-through */
820 continue;
821 }
822 }
823
824 return true;
825}
826
827int lightrec_compile_block(struct block *block)
828{
829 struct lightrec_state *state = block->state;
830 bool op_list_freed = false, fully_tagged = false;
831 struct opcode *elm;
832 jit_state_t *_jit;
833 jit_node_t *start_of_block;
834 bool skip_next = false;
835 jit_word_t code_size;
836 unsigned int i, j;
837 u32 next_pc;
838
839 fully_tagged = lightrec_block_is_fully_tagged(block);
840 if (fully_tagged)
841 block->flags |= BLOCK_FULLY_TAGGED;
842
843 _jit = jit_new_state();
844 if (!_jit)
845 return -ENOMEM;
846
847 block->_jit = _jit;
848
849 lightrec_regcache_reset(state->reg_cache);
850 state->cycles = 0;
851 state->nb_branches = 0;
852 state->nb_local_branches = 0;
853 state->nb_targets = 0;
854
855 jit_prolog();
856 jit_tramp(256);
857
858 start_of_block = jit_label();
859
860 for (elm = block->opcode_list; elm; elm = elm->next) {
861 next_pc = block->pc + elm->offset * sizeof(u32);
862
863 if (skip_next) {
864 skip_next = false;
865 continue;
866 }
867
868 state->cycles += lightrec_cycles_of_opcode(elm->c);
869
870 if (elm->flags & LIGHTREC_EMULATE_BRANCH) {
871 pr_debug("Branch at offset 0x%x will be emulated\n",
872 elm->offset << 2);
873 lightrec_emit_eob(block, elm, next_pc);
874 skip_next = !(elm->flags & LIGHTREC_NO_DS);
875 } else if (elm->opcode) {
876 lightrec_rec_opcode(block, elm, next_pc);
877 skip_next = has_delay_slot(elm->c) &&
878 !(elm->flags & LIGHTREC_NO_DS);
879#if _WIN32
880 /* FIXME: GNU Lightning on Windows seems to use our
881 * mapped registers as temporaries. Until the actual bug
882 * is found and fixed, unconditionally mark our
883 * registers as live here. */
884 lightrec_regcache_mark_live(state->reg_cache, _jit);
885#endif
886 }
887 }
888
889 for (i = 0; i < state->nb_branches; i++)
890 jit_patch(state->branches[i]);
891
892 for (i = 0; i < state->nb_local_branches; i++) {
893 struct lightrec_branch *branch = &state->local_branches[i];
894
895 pr_debug("Patch local branch to offset 0x%x\n",
896 branch->target << 2);
897
898 if (branch->target == 0) {
899 jit_patch_at(branch->branch, start_of_block);
900 continue;
901 }
902
903 for (j = 0; j < state->nb_targets; j++) {
904 if (state->targets[j].offset == branch->target) {
905 jit_patch_at(branch->branch,
906 state->targets[j].label);
907 break;
908 }
909 }
910
911 if (j == state->nb_targets)
912 pr_err("Unable to find branch target\n");
913 }
914
915 jit_ldxi(JIT_R0, LIGHTREC_REG_STATE,
916 offsetof(struct lightrec_state, eob_wrapper_func));
917
918 jit_jmpr(JIT_R0);
919
920 jit_ret();
921 jit_epilog();
922
923 block->function = jit_emit();
924
925 /* Add compiled function to the LUT */
926 state->code_lut[lut_offset(block->pc)] = block->function;
927
928 jit_get_code(&code_size);
929 lightrec_register(MEM_FOR_CODE, code_size);
930
931 block->code_size = code_size;
932
933 if (ENABLE_DISASSEMBLER) {
934 pr_debug("Compiling block at PC: 0x%x\n", block->pc);
935 jit_disassemble();
936 }
937
938 jit_clear_state();
939
940#if ENABLE_THREADED_COMPILER
941 if (fully_tagged)
942 op_list_freed = atomic_flag_test_and_set(&block->op_list_freed);
943#endif
944 if (fully_tagged && !op_list_freed) {
945 pr_debug("Block PC 0x%08x is fully tagged"
946 " - free opcode list\n", block->pc);
947 lightrec_free_opcode_list(state, block->opcode_list);
948 block->opcode_list = NULL;
949 }
950
951 return 0;
952}
953
954u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle)
955{
956 s32 (*func)(void *, s32) = (void *)state->dispatcher->function;
957 void *block_trace;
958 s32 cycles_delta;
959
960 state->exit_flags = LIGHTREC_EXIT_NORMAL;
961
962 /* Handle the cycle counter overflowing */
963 if (unlikely(target_cycle < state->current_cycle))
964 target_cycle = UINT_MAX;
965
966 state->target_cycle = target_cycle;
967
968 block_trace = get_next_block_func(state, pc);
969 if (block_trace) {
970 cycles_delta = state->target_cycle - state->current_cycle;
971
972 cycles_delta = (*func)(block_trace, cycles_delta);
973
974 state->current_cycle = state->target_cycle - cycles_delta;
975 }
976
977 return state->next_pc;
978}
979
980u32 lightrec_execute_one(struct lightrec_state *state, u32 pc)
981{
982 return lightrec_execute(state, pc, state->current_cycle);
983}
984
985u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc)
986{
987 struct block *block = lightrec_get_block(state, pc);
988 if (!block)
989 return 0;
990
991 state->exit_flags = LIGHTREC_EXIT_NORMAL;
992
993 return lightrec_emulate_block(block, pc);
994}
995
996void lightrec_free_block(struct block *block)
997{
998 lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32));
999 if (block->opcode_list)
1000 lightrec_free_opcode_list(block->state, block->opcode_list);
1001 if (block->_jit)
1002 _jit_destroy_state(block->_jit);
1003 lightrec_unregister(MEM_FOR_CODE, block->code_size);
1004 lightrec_free(block->state, MEM_FOR_IR, sizeof(*block), block);
1005}
1006
1007struct lightrec_state * lightrec_init(char *argv0,
1008 const struct lightrec_mem_map *map,
1009 size_t nb,
1010 const struct lightrec_ops *ops)
1011{
1012 struct lightrec_state *state;
1013
1014 /* Sanity-check ops */
1015 if (!ops ||
1016 !ops->cop0_ops.mfc || !ops->cop0_ops.cfc || !ops->cop0_ops.mtc ||
1017 !ops->cop0_ops.ctc || !ops->cop0_ops.op ||
1018 !ops->cop2_ops.mfc || !ops->cop2_ops.cfc || !ops->cop2_ops.mtc ||
1019 !ops->cop2_ops.ctc || !ops->cop2_ops.op) {
1020 pr_err("Missing callbacks in lightrec_ops structure\n");
1021 return NULL;
1022 }
1023
1024 init_jit(argv0);
1025
1026 state = calloc(1, sizeof(*state) +
1027 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1028 if (!state)
1029 goto err_finish_jit;
1030
1031 lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) +
1032 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1033
1034#if ENABLE_TINYMM
1035 state->tinymm = tinymm_init(malloc, free, 4096);
1036 if (!state->tinymm)
1037 goto err_free_state;
1038#endif
1039
1040 state->block_cache = lightrec_blockcache_init(state);
1041 if (!state->block_cache)
1042 goto err_free_tinymm;
1043
1044 state->reg_cache = lightrec_regcache_init(state);
1045 if (!state->reg_cache)
1046 goto err_free_block_cache;
1047
1048 if (ENABLE_THREADED_COMPILER) {
1049 state->rec = lightrec_recompiler_init(state);
1050 if (!state->rec)
1051 goto err_free_reg_cache;
1052 }
1053
1054 state->nb_maps = nb;
1055 state->maps = map;
1056
1057 memcpy(&state->ops, ops, sizeof(*ops));
1058
1059 state->dispatcher = generate_dispatcher(state);
1060 if (!state->dispatcher)
1061 goto err_free_recompiler;
1062
1063 state->rw_generic_wrapper = generate_wrapper(state,
1064 lightrec_rw_generic_cb,
1065 true);
1066 if (!state->rw_generic_wrapper)
1067 goto err_free_dispatcher;
1068
1069 state->rw_wrapper = generate_wrapper(state, lightrec_rw_cb, false);
1070 if (!state->rw_wrapper)
1071 goto err_free_generic_rw_wrapper;
1072
1073 state->mfc_wrapper = generate_wrapper(state, lightrec_mfc_cb, false);
1074 if (!state->mfc_wrapper)
1075 goto err_free_rw_wrapper;
1076
1077 state->mtc_wrapper = generate_wrapper(state, lightrec_mtc_cb, false);
1078 if (!state->mtc_wrapper)
1079 goto err_free_mfc_wrapper;
1080
1081 state->rfe_wrapper = generate_wrapper(state, lightrec_rfe_cb, false);
1082 if (!state->rfe_wrapper)
1083 goto err_free_mtc_wrapper;
1084
1085 state->cp_wrapper = generate_wrapper(state, lightrec_cp_cb, false);
1086 if (!state->cp_wrapper)
1087 goto err_free_rfe_wrapper;
1088
1089 state->syscall_wrapper = generate_wrapper(state, lightrec_syscall_cb,
1090 false);
1091 if (!state->syscall_wrapper)
1092 goto err_free_cp_wrapper;
1093
1094 state->break_wrapper = generate_wrapper(state, lightrec_break_cb,
1095 false);
1096 if (!state->break_wrapper)
1097 goto err_free_syscall_wrapper;
1098
1099 state->rw_generic_func = state->rw_generic_wrapper->function;
1100 state->rw_func = state->rw_wrapper->function;
1101 state->mfc_func = state->mfc_wrapper->function;
1102 state->mtc_func = state->mtc_wrapper->function;
1103 state->rfe_func = state->rfe_wrapper->function;
1104 state->cp_func = state->cp_wrapper->function;
1105 state->syscall_func = state->syscall_wrapper->function;
1106 state->break_func = state->break_wrapper->function;
1107
1108 map = &state->maps[PSX_MAP_BIOS];
1109 state->offset_bios = (uintptr_t)map->address - map->pc;
1110
1111 map = &state->maps[PSX_MAP_SCRATCH_PAD];
1112 state->offset_scratch = (uintptr_t)map->address - map->pc;
1113
1114 map = &state->maps[PSX_MAP_KERNEL_USER_RAM];
1115 state->offset_ram = (uintptr_t)map->address - map->pc;
1116
1117 if (state->maps[PSX_MAP_MIRROR1].address == map->address + 0x200000 &&
1118 state->maps[PSX_MAP_MIRROR2].address == map->address + 0x400000 &&
1119 state->maps[PSX_MAP_MIRROR3].address == map->address + 0x600000)
1120 state->mirrors_mapped = true;
1121
1122 return state;
1123
1124err_free_syscall_wrapper:
1125 lightrec_free_block(state->syscall_wrapper);
1126err_free_cp_wrapper:
1127 lightrec_free_block(state->cp_wrapper);
1128err_free_rfe_wrapper:
1129 lightrec_free_block(state->rfe_wrapper);
1130err_free_mtc_wrapper:
1131 lightrec_free_block(state->mtc_wrapper);
1132err_free_mfc_wrapper:
1133 lightrec_free_block(state->mfc_wrapper);
1134err_free_rw_wrapper:
1135 lightrec_free_block(state->rw_wrapper);
1136err_free_generic_rw_wrapper:
1137 lightrec_free_block(state->rw_generic_wrapper);
1138err_free_dispatcher:
1139 lightrec_free_block(state->dispatcher);
1140err_free_recompiler:
1141 if (ENABLE_THREADED_COMPILER)
1142 lightrec_free_recompiler(state->rec);
1143err_free_reg_cache:
1144 lightrec_free_regcache(state->reg_cache);
1145err_free_block_cache:
1146 lightrec_free_block_cache(state->block_cache);
1147err_free_tinymm:
1148#if ENABLE_TINYMM
1149 tinymm_shutdown(state->tinymm);
1150err_free_state:
1151#endif
1152 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1153 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1154 free(state);
1155err_finish_jit:
1156 finish_jit();
1157 return NULL;
1158}
1159
1160void lightrec_destroy(struct lightrec_state *state)
1161{
1162 if (ENABLE_THREADED_COMPILER)
1163 lightrec_free_recompiler(state->rec);
1164
1165 lightrec_free_regcache(state->reg_cache);
1166 lightrec_free_block_cache(state->block_cache);
1167 lightrec_free_block(state->dispatcher);
1168 lightrec_free_block(state->rw_generic_wrapper);
1169 lightrec_free_block(state->rw_wrapper);
1170 lightrec_free_block(state->mfc_wrapper);
1171 lightrec_free_block(state->mtc_wrapper);
1172 lightrec_free_block(state->rfe_wrapper);
1173 lightrec_free_block(state->cp_wrapper);
1174 lightrec_free_block(state->syscall_wrapper);
1175 lightrec_free_block(state->break_wrapper);
1176 finish_jit();
1177
1178#if ENABLE_TINYMM
1179 tinymm_shutdown(state->tinymm);
1180#endif
1181 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1182 sizeof(*state->code_lut) * CODE_LUT_SIZE);
1183 free(state);
1184}
1185
1186void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len)
1187{
1188 u32 kaddr = kunseg(addr & ~0x3);
1189 const struct lightrec_mem_map *map = lightrec_get_map(state, kaddr);
1190
1191 if (map) {
1192 while (map->mirror_of)
1193 map = map->mirror_of;
1194
1195 if (map != &state->maps[PSX_MAP_KERNEL_USER_RAM])
1196 return;
1197
1198 /* Handle mirrors */
1199 kaddr &= (state->maps[PSX_MAP_KERNEL_USER_RAM].length - 1);
1200
1201 for (; len > 4; len -= 4, kaddr += 4)
1202 lightrec_invalidate_map(state, map, kaddr);
1203
1204 lightrec_invalidate_map(state, map, kaddr);
1205 }
1206}
1207
1208void lightrec_invalidate_all(struct lightrec_state *state)
1209{
1210 memset(state->code_lut, 0, sizeof(*state->code_lut) * CODE_LUT_SIZE);
1211}
1212
1213void lightrec_set_invalidate_mode(struct lightrec_state *state, bool dma_only)
1214{
1215 if (state->invalidate_from_dma_only != dma_only)
1216 lightrec_invalidate_all(state);
1217
1218 state->invalidate_from_dma_only = dma_only;
1219}
1220
1221void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags)
1222{
1223 if (flags != LIGHTREC_EXIT_NORMAL) {
1224 state->exit_flags |= flags;
1225 state->target_cycle = state->current_cycle;
1226 }
1227}
1228
1229u32 lightrec_exit_flags(struct lightrec_state *state)
1230{
1231 return state->exit_flags;
1232}
1233
1234void lightrec_dump_registers(struct lightrec_state *state, u32 regs[34])
1235{
1236 memcpy(regs, state->native_reg_cache, sizeof(state->native_reg_cache));
1237}
1238
1239void lightrec_restore_registers(struct lightrec_state *state, u32 regs[34])
1240{
1241 memcpy(state->native_reg_cache, regs, sizeof(state->native_reg_cache));
1242}
1243
1244u32 lightrec_current_cycle_count(const struct lightrec_state *state)
1245{
1246 return state->current_cycle;
1247}
1248
1249void lightrec_reset_cycle_count(struct lightrec_state *state, u32 cycles)
1250{
1251 state->current_cycle = cycles;
1252
1253 if (state->target_cycle < cycles)
1254 state->target_cycle = cycles;
1255}
1256
1257void lightrec_set_target_cycle_count(struct lightrec_state *state, u32 cycles)
1258{
1259 if (state->exit_flags == LIGHTREC_EXIT_NORMAL) {
1260 if (cycles < state->current_cycle)
1261 cycles = state->current_cycle;
1262
1263 state->target_cycle = cycles;
1264 }
1265}
ARM optimized PCSX fork