7fdf74a34d1eeeb2710699bf5e2fb7db6fb5c9bd
[pcsx_rearmed.git] / deps / lightrec / lightrec.c
1 /*
2  * Copyright (C) 2014-2020 Paul Cercueil <paul@crapouillou.net>
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  */
14
15 #include "blockcache.h"
16 #include "config.h"
17 #include "debug.h"
18 #include "disassembler.h"
19 #include "emitter.h"
20 #include "interpreter.h"
21 #include "lightrec.h"
22 #include "memmanager.h"
23 #include "reaper.h"
24 #include "recompiler.h"
25 #include "regcache.h"
26 #include "optimizer.h"
27
28 #include <errno.h>
29 #include <lightning.h>
30 #include <limits.h>
31 #if ENABLE_THREADED_COMPILER
32 #include <stdatomic.h>
33 #endif
34 #include <stdbool.h>
35 #include <stddef.h>
36 #include <string.h>
37 #if ENABLE_TINYMM
38 #include <tinymm.h>
39 #endif
40
41 #define GENMASK(h, l) \
42         (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
43
44 static struct block * lightrec_precompile_block(struct lightrec_state *state,
45                                                 u32 pc);
46
47 static void lightrec_default_sb(struct lightrec_state *state, u32 opcode,
48                                 void *host, u32 addr, u8 data)
49 {
50         *(u8 *)host = data;
51
52         if (!state->invalidate_from_dma_only)
53                 lightrec_invalidate(state, addr, 1);
54 }
55
56 static void lightrec_default_sh(struct lightrec_state *state, u32 opcode,
57                                 void *host, u32 addr, u16 data)
58 {
59         *(u16 *)host = HTOLE16(data);
60
61         if (!state->invalidate_from_dma_only)
62                 lightrec_invalidate(state, addr, 2);
63 }
64
65 static void lightrec_default_sw(struct lightrec_state *state, u32 opcode,
66                                 void *host, u32 addr, u32 data)
67 {
68         *(u32 *)host = HTOLE32(data);
69
70         if (!state->invalidate_from_dma_only)
71                 lightrec_invalidate(state, addr, 4);
72 }
73
74 static u8 lightrec_default_lb(struct lightrec_state *state,
75                               u32 opcode, void *host, u32 addr)
76 {
77         return *(u8 *)host;
78 }
79
80 static u16 lightrec_default_lh(struct lightrec_state *state,
81                                u32 opcode, void *host, u32 addr)
82 {
83         return LE16TOH(*(u16 *)host);
84 }
85
86 static u32 lightrec_default_lw(struct lightrec_state *state,
87                                u32 opcode, void *host, u32 addr)
88 {
89         return LE32TOH(*(u32 *)host);
90 }
91
92 static const struct lightrec_mem_map_ops lightrec_default_ops = {
93         .sb = lightrec_default_sb,
94         .sh = lightrec_default_sh,
95         .sw = lightrec_default_sw,
96         .lb = lightrec_default_lb,
97         .lh = lightrec_default_lh,
98         .lw = lightrec_default_lw,
99 };
100
101 static void __segfault_cb(struct lightrec_state *state, u32 addr)
102 {
103         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
104         pr_err("Segmentation fault in recompiled code: invalid "
105                "load/store at address 0x%08x\n", addr);
106 }
107
108 static void lightrec_swl(struct lightrec_state *state,
109                          const struct lightrec_mem_map_ops *ops,
110                          u32 opcode, void *host, u32 addr, u32 data)
111 {
112         unsigned int shift = addr & 0x3;
113         unsigned int mask = GENMASK(31, (shift + 1) * 8);
114         u32 old_data;
115
116         /* Align to 32 bits */
117         addr &= ~3;
118         host = (void *)((uintptr_t)host & ~3);
119
120         old_data = ops->lw(state, opcode, host, addr);
121
122         data = (data >> ((3 - shift) * 8)) | (old_data & mask);
123
124         ops->sw(state, opcode, host, addr, data);
125 }
126
127 static void lightrec_swr(struct lightrec_state *state,
128                          const struct lightrec_mem_map_ops *ops,
129                          u32 opcode, void *host, u32 addr, u32 data)
130 {
131         unsigned int shift = addr & 0x3;
132         unsigned int mask = (1 << (shift * 8)) - 1;
133         u32 old_data;
134
135         /* Align to 32 bits */
136         addr &= ~3;
137         host = (void *)((uintptr_t)host & ~3);
138
139         old_data = ops->lw(state, opcode, host, addr);
140
141         data = (data << (shift * 8)) | (old_data & mask);
142
143         ops->sw(state, opcode, host, addr, data);
144 }
145
146 static void lightrec_swc2(struct lightrec_state *state, union code op,
147                           const struct lightrec_mem_map_ops *ops,
148                           void *host, u32 addr)
149 {
150         u32 data = state->ops.cop2_ops.mfc(state, op.opcode, op.i.rt);
151
152         ops->sw(state, op.opcode, host, addr, data);
153 }
154
155 static u32 lightrec_lwl(struct lightrec_state *state,
156                         const struct lightrec_mem_map_ops *ops,
157                         u32 opcode, void *host, u32 addr, u32 data)
158 {
159         unsigned int shift = addr & 0x3;
160         unsigned int mask = (1 << (24 - shift * 8)) - 1;
161         u32 old_data;
162
163         /* Align to 32 bits */
164         addr &= ~3;
165         host = (void *)((uintptr_t)host & ~3);
166
167         old_data = ops->lw(state, opcode, host, addr);
168
169         return (data & mask) | (old_data << (24 - shift * 8));
170 }
171
172 static u32 lightrec_lwr(struct lightrec_state *state,
173                         const struct lightrec_mem_map_ops *ops,
174                         u32 opcode, void *host, u32 addr, u32 data)
175 {
176         unsigned int shift = addr & 0x3;
177         unsigned int mask = GENMASK(31, 32 - shift * 8);
178         u32 old_data;
179
180         /* Align to 32 bits */
181         addr &= ~3;
182         host = (void *)((uintptr_t)host & ~3);
183
184         old_data = ops->lw(state, opcode, host, addr);
185
186         return (data & mask) | (old_data >> (shift * 8));
187 }
188
189 static void lightrec_lwc2(struct lightrec_state *state, union code op,
190                           const struct lightrec_mem_map_ops *ops,
191                           void *host, u32 addr)
192 {
193         u32 data = ops->lw(state, op.opcode, host, addr);
194
195         state->ops.cop2_ops.mtc(state, op.opcode, op.i.rt, data);
196 }
197
198 static void lightrec_invalidate_map(struct lightrec_state *state,
199                 const struct lightrec_mem_map *map, u32 addr)
200 {
201         if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM])
202                 state->code_lut[lut_offset(addr)] = NULL;
203 }
204
205 static const struct lightrec_mem_map *
206 lightrec_get_map(struct lightrec_state *state, u32 kaddr)
207 {
208         unsigned int i;
209
210         for (i = 0; i < state->nb_maps; i++) {
211                 const struct lightrec_mem_map *map = &state->maps[i];
212
213                 if (kaddr >= map->pc && kaddr < map->pc + map->length)
214                         return map;
215         }
216
217         return NULL;
218 }
219
220 u32 lightrec_rw(struct lightrec_state *state, union code op,
221                 u32 addr, u32 data, u16 *flags)
222 {
223         const struct lightrec_mem_map *map;
224         const struct lightrec_mem_map_ops *ops;
225         u32 kaddr, pc, opcode = op.opcode;
226         void *host;
227
228         addr += (s16) op.i.imm;
229         kaddr = kunseg(addr);
230
231         map = lightrec_get_map(state, kaddr);
232         if (!map) {
233                 __segfault_cb(state, addr);
234                 return 0;
235         }
236
237         pc = map->pc;
238
239         while (map->mirror_of)
240                 map = map->mirror_of;
241
242         host = (void *)((uintptr_t)map->address + kaddr - pc);
243
244         if (unlikely(map->ops)) {
245                 if (flags)
246                         *flags |= LIGHTREC_HW_IO;
247
248                 ops = map->ops;
249         } else {
250                 if (flags)
251                         *flags |= LIGHTREC_DIRECT_IO;
252
253                 ops = &lightrec_default_ops;
254         }
255
256         switch (op.i.op) {
257         case OP_SB:
258                 ops->sb(state, opcode, host, addr, (u8) data);
259                 return 0;
260         case OP_SH:
261                 ops->sh(state, opcode, host, addr, (u16) data);
262                 return 0;
263         case OP_SWL:
264                 lightrec_swl(state, ops, opcode, host, addr, data);
265                 return 0;
266         case OP_SWR:
267                 lightrec_swr(state, ops, opcode, host, addr, data);
268                 return 0;
269         case OP_SW:
270                 ops->sw(state, opcode, host, addr, data);
271                 return 0;
272         case OP_SWC2:
273                 lightrec_swc2(state, op, ops, host, addr);
274                 return 0;
275         case OP_LB:
276                 return (s32) (s8) ops->lb(state, opcode, host, addr);
277         case OP_LBU:
278                 return ops->lb(state, opcode, host, addr);
279         case OP_LH:
280                 return (s32) (s16) ops->lh(state, opcode, host, addr);
281         case OP_LHU:
282                 return ops->lh(state, opcode, host, addr);
283         case OP_LWC2:
284                 lightrec_lwc2(state, op, ops, host, addr);
285                 return 0;
286         case OP_LWL:
287                 return lightrec_lwl(state, ops, opcode, host, addr, data);
288         case OP_LWR:
289                 return lightrec_lwr(state, ops, opcode, host, addr, data);
290         case OP_LW:
291         default:
292                 return ops->lw(state, opcode, host, addr);
293         }
294 }
295
296 static void lightrec_rw_helper(struct lightrec_state *state,
297                                union code op, u16 *flags)
298 {
299         u32 ret = lightrec_rw(state, op,
300                           state->native_reg_cache[op.i.rs],
301                           state->native_reg_cache[op.i.rt], flags);
302
303         switch (op.i.op) {
304         case OP_LB:
305         case OP_LBU:
306         case OP_LH:
307         case OP_LHU:
308         case OP_LWL:
309         case OP_LWR:
310         case OP_LW:
311                 if (op.i.rt)
312                         state->native_reg_cache[op.i.rt] = ret;
313         default: /* fall-through */
314                 break;
315         }
316 }
317
318 static void lightrec_rw_cb(struct lightrec_state *state, union code op)
319 {
320         lightrec_rw_helper(state, op, NULL);
321 }
322
323 static void lightrec_rw_generic_cb(struct lightrec_state *state,
324                                    struct opcode *op, struct block *block)
325 {
326         bool was_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
327
328         lightrec_rw_helper(state, op->c, &op->flags);
329
330         if (!was_tagged) {
331                 pr_debug("Opcode of block at PC 0x%08x offset 0x%x has been "
332                          "tagged - flag for recompilation\n",
333                          block->pc, op->offset << 2);
334
335                 block->flags |= BLOCK_SHOULD_RECOMPILE;
336         }
337 }
338
339 u32 lightrec_mfc(struct lightrec_state *state, union code op)
340 {
341         bool is_cfc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CFC0) ||
342                       (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CFC2);
343         u32 (*func)(struct lightrec_state *, u32, u8);
344         const struct lightrec_cop_ops *ops;
345
346         if (op.i.op == OP_CP0)
347                 ops = &state->ops.cop0_ops;
348         else
349                 ops = &state->ops.cop2_ops;
350
351         if (is_cfc)
352                 func = ops->cfc;
353         else
354                 func = ops->mfc;
355
356         return (*func)(state, op.opcode, op.r.rd);
357 }
358
359 static void lightrec_mfc_cb(struct lightrec_state *state, union code op)
360 {
361         u32 rt = lightrec_mfc(state, op);
362
363         if (op.r.rt)
364                 state->native_reg_cache[op.r.rt] = rt;
365 }
366
367 void lightrec_mtc(struct lightrec_state *state, union code op, u32 data)
368 {
369         bool is_ctc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CTC0) ||
370                       (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CTC2);
371         void (*func)(struct lightrec_state *, u32, u8, u32);
372         const struct lightrec_cop_ops *ops;
373
374         if (op.i.op == OP_CP0)
375                 ops = &state->ops.cop0_ops;
376         else
377                 ops = &state->ops.cop2_ops;
378
379         if (is_ctc)
380                 func = ops->ctc;
381         else
382                 func = ops->mtc;
383
384         (*func)(state, op.opcode, op.r.rd, data);
385 }
386
387 static void lightrec_mtc_cb(struct lightrec_state *state, union code op)
388 {
389         lightrec_mtc(state, op, state->native_reg_cache[op.r.rt]);
390 }
391
392 static void lightrec_rfe_cb(struct lightrec_state *state, union code op)
393 {
394         u32 status;
395
396         /* Read CP0 Status register (r12) */
397         status = state->ops.cop0_ops.mfc(state, op.opcode, 12);
398
399         /* Switch the bits */
400         status = ((status & 0x3c) >> 2) | (status & ~0xf);
401
402         /* Write it back */
403         state->ops.cop0_ops.ctc(state, op.opcode, 12, status);
404 }
405
406 static void lightrec_cp_cb(struct lightrec_state *state, union code op)
407 {
408         void (*func)(struct lightrec_state *, u32);
409
410         if ((op.opcode >> 25) & 1)
411                 func = state->ops.cop2_ops.op;
412         else
413                 func = state->ops.cop0_ops.op;
414
415         (*func)(state, op.opcode);
416 }
417
418 static void lightrec_syscall_cb(struct lightrec_state *state, union code op)
419 {
420         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SYSCALL);
421 }
422
423 static void lightrec_break_cb(struct lightrec_state *state, union code op)
424 {
425         lightrec_set_exit_flags(state, LIGHTREC_EXIT_BREAK);
426 }
427
428 struct block * lightrec_get_block(struct lightrec_state *state, u32 pc)
429 {
430         struct block *block = lightrec_find_block(state->block_cache, pc);
431
432         if (block && lightrec_block_is_outdated(block)) {
433                 pr_debug("Block at PC 0x%08x is outdated!\n", block->pc);
434
435                 /* Make sure the recompiler isn't processing the block we'll
436                  * destroy */
437                 if (ENABLE_THREADED_COMPILER)
438                         lightrec_recompiler_remove(state->rec, block);
439
440                 lightrec_unregister_block(state->block_cache, block);
441                 remove_from_code_lut(state->block_cache, block);
442                 lightrec_free_block(block);
443                 block = NULL;
444         }
445
446         if (!block) {
447                 block = lightrec_precompile_block(state, pc);
448                 if (!block) {
449                         pr_err("Unable to recompile block at PC 0x%x\n", pc);
450                         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
451                         return NULL;
452                 }
453
454                 lightrec_register_block(state->block_cache, block);
455         }
456
457         return block;
458 }
459
460 static void * get_next_block_func(struct lightrec_state *state, u32 pc)
461 {
462         struct block *block;
463         bool should_recompile;
464         void *func;
465
466         for (;;) {
467                 func = state->code_lut[lut_offset(pc)];
468                 if (func && func != state->get_next_block)
469                         return func;
470
471                 block = lightrec_get_block(state, pc);
472
473                 if (unlikely(!block))
474                         return NULL;
475
476                 should_recompile = block->flags & BLOCK_SHOULD_RECOMPILE &&
477                         !(block->flags & BLOCK_IS_DEAD);
478
479                 if (unlikely(should_recompile)) {
480                         pr_debug("Block at PC 0x%08x should recompile\n", pc);
481
482                         lightrec_unregister(MEM_FOR_CODE, block->code_size);
483
484                         if (ENABLE_THREADED_COMPILER)
485                                 lightrec_recompiler_add(state->rec, block);
486                         else
487                                 lightrec_compile_block(block);
488                 }
489
490                 if (ENABLE_THREADED_COMPILER && likely(!should_recompile))
491                         func = lightrec_recompiler_run_first_pass(block, &pc);
492                 else
493                         func = block->function;
494
495                 if (likely(func))
496                         return func;
497
498                 /* Block wasn't compiled yet - run the interpreter */
499                 if (!ENABLE_THREADED_COMPILER &&
500                     ((ENABLE_FIRST_PASS && likely(!should_recompile)) ||
501                      unlikely(block->flags & BLOCK_NEVER_COMPILE)))
502                         pc = lightrec_emulate_block(block, pc);
503
504                 if (likely(!(block->flags & BLOCK_NEVER_COMPILE))) {
505                         /* Then compile it using the profiled data */
506                         if (ENABLE_THREADED_COMPILER)
507                                 lightrec_recompiler_add(state->rec, block);
508                         else
509                                 lightrec_compile_block(block);
510                 }
511
512                 if (state->exit_flags != LIGHTREC_EXIT_NORMAL ||
513                     state->current_cycle >= state->target_cycle) {
514                         state->next_pc = pc;
515                         return NULL;
516                 }
517         }
518 }
519
520 static s32 c_generic_function_wrapper(struct lightrec_state *state,
521                                       s32 cycles_delta,
522                                       void (*f)(struct lightrec_state *,
523                                                 struct opcode *,
524                                                 struct block *),
525                                       struct opcode *op, struct block *block)
526 {
527         state->current_cycle = state->target_cycle - cycles_delta;
528
529         (*f)(state, op, block);
530
531         return state->target_cycle - state->current_cycle;
532 }
533
534 static s32 c_function_wrapper(struct lightrec_state *state, s32 cycles_delta,
535                               void (*f)(struct lightrec_state *, union code),
536                               union code op)
537 {
538         state->current_cycle = state->target_cycle - cycles_delta;
539
540         (*f)(state, op);
541
542         return state->target_cycle - state->current_cycle;
543 }
544
545 static struct block * generate_wrapper(struct lightrec_state *state,
546                                        void *f, bool generic)
547 {
548         struct block *block;
549         jit_state_t *_jit;
550         unsigned int i;
551         int stack_ptr;
552         jit_word_t code_size;
553         jit_node_t *to_tramp, *to_fn_epilog;
554
555         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
556         if (!block)
557                 goto err_no_mem;
558
559         _jit = jit_new_state();
560         if (!_jit)
561                 goto err_free_block;
562
563         jit_name("RW wrapper");
564         jit_note(__FILE__, __LINE__);
565
566         /* Wrapper entry point */
567         jit_prolog();
568
569         stack_ptr = jit_allocai(sizeof(uintptr_t) * NUM_TEMPS);
570
571         for (i = 0; i < NUM_TEMPS; i++)
572                 jit_stxi(stack_ptr + i * sizeof(uintptr_t), JIT_FP, JIT_R(i));
573
574         /* Jump to the trampoline */
575         to_tramp = jit_jmpi();
576
577         /* The trampoline will jump back here */
578         to_fn_epilog = jit_label();
579
580         for (i = 0; i < NUM_TEMPS; i++)
581                 jit_ldxi(JIT_R(i), JIT_FP, stack_ptr + i * sizeof(uintptr_t));
582
583         jit_ret();
584         jit_epilog();
585
586         /* Trampoline entry point.
587          * The sole purpose of the trampoline is to cheese Lightning not to
588          * save/restore the callee-saved register LIGHTREC_REG_CYCLE, since we
589          * do want to return to the caller with this register modified. */
590         jit_prolog();
591         jit_tramp(256);
592         jit_patch(to_tramp);
593
594         jit_prepare();
595         jit_pushargr(LIGHTREC_REG_STATE);
596         jit_pushargr(LIGHTREC_REG_CYCLE);
597         jit_pushargi((uintptr_t)f);
598         jit_pushargr(JIT_R0);
599         if (generic) {
600                 jit_pushargr(JIT_R1);
601                 jit_finishi(c_generic_function_wrapper);
602         } else {
603                 jit_finishi(c_function_wrapper);
604         }
605
606 #if __WORDSIZE == 64
607         jit_retval_i(LIGHTREC_REG_CYCLE);
608 #else
609         jit_retval(LIGHTREC_REG_CYCLE);
610 #endif
611
612         jit_patch_at(jit_jmpi(), to_fn_epilog);
613         jit_epilog();
614
615         block->state = state;
616         block->_jit = _jit;
617         block->function = jit_emit();
618         block->opcode_list = NULL;
619         block->flags = 0;
620         block->nb_ops = 0;
621
622         jit_get_code(&code_size);
623         lightrec_register(MEM_FOR_CODE, code_size);
624
625         block->code_size = code_size;
626
627         if (ENABLE_DISASSEMBLER) {
628                 pr_debug("Wrapper block:\n");
629                 jit_disassemble();
630         }
631
632         jit_clear_state();
633         return block;
634
635 err_free_block:
636         lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
637 err_no_mem:
638         pr_err("Unable to compile wrapper: Out of memory\n");
639         return NULL;
640 }
641
642 static struct block * generate_dispatcher(struct lightrec_state *state)
643 {
644         struct block *block;
645         jit_state_t *_jit;
646         jit_node_t *to_end, *to_end2, *to_c, *loop, *addr, *addr2;
647         unsigned int i;
648         u32 offset, ram_len;
649         jit_word_t code_size;
650
651         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
652         if (!block)
653                 goto err_no_mem;
654
655         _jit = jit_new_state();
656         if (!_jit)
657                 goto err_free_block;
658
659         jit_name("dispatcher");
660         jit_note(__FILE__, __LINE__);
661
662         jit_prolog();
663         jit_frame(256);
664
665         jit_getarg(JIT_R0, jit_arg());
666 #if __WORDSIZE == 64
667         jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg());
668 #else
669         jit_getarg(LIGHTREC_REG_CYCLE, jit_arg());
670 #endif
671
672         /* Force all callee-saved registers to be pushed on the stack */
673         for (i = 0; i < NUM_REGS; i++)
674                 jit_movr(JIT_V(i), JIT_V(i));
675
676         /* Pass lightrec_state structure to blocks, using the last callee-saved
677          * register that Lightning provides */
678         jit_movi(LIGHTREC_REG_STATE, (intptr_t) state);
679
680         loop = jit_label();
681
682         /* Call the block's code */
683         jit_jmpr(JIT_R0);
684
685         /* The block will jump here, with the number of cycles remaining in
686          * LIGHTREC_REG_CYCLE */
687         addr2 = jit_indirect();
688
689         /* Jump to end if state->target_cycle < state->current_cycle */
690         to_end = jit_blei(LIGHTREC_REG_CYCLE, 0);
691
692         /* Convert next PC to KUNSEG and avoid mirrors */
693         ram_len = state->maps[PSX_MAP_KERNEL_USER_RAM].length;
694         jit_andi(JIT_R0, JIT_V0, 0x10000000 | (ram_len - 1));
695         to_c = jit_bgei(JIT_R0, ram_len);
696
697         /* Fast path: code is running from RAM, use the code LUT */
698 #if __WORDSIZE == 64
699         jit_lshi(JIT_R0, JIT_R0, 1);
700 #endif
701         jit_addr(JIT_R0, JIT_R0, LIGHTREC_REG_STATE);
702         jit_ldxi(JIT_R0, JIT_R0, offsetof(struct lightrec_state, code_lut));
703
704         /* If we get non-NULL, loop */
705         jit_patch_at(jit_bnei(JIT_R0, 0), loop);
706
707         /* Slow path: call C function get_next_block_func() */
708         jit_patch(to_c);
709
710         if (ENABLE_FIRST_PASS) {
711                 /* We may call the interpreter - update state->current_cycle */
712                 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
713                            offsetof(struct lightrec_state, target_cycle));
714                 jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE);
715                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
716                            LIGHTREC_REG_STATE, JIT_R1);
717         }
718
719         /* The code LUT will be set to this address when the block at the target
720          * PC has been preprocessed but not yet compiled by the threaded
721          * recompiler */
722         addr = jit_indirect();
723
724         /* Get the next block */
725         jit_prepare();
726         jit_pushargr(LIGHTREC_REG_STATE);
727         jit_pushargr(JIT_V0);
728         jit_finishi(&get_next_block_func);
729         jit_retval(JIT_R0);
730
731         if (ENABLE_FIRST_PASS) {
732                 /* The interpreter may have updated state->current_cycle and
733                  * state->target_cycle - recalc the delta */
734                 jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE,
735                            offsetof(struct lightrec_state, current_cycle));
736                 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
737                            offsetof(struct lightrec_state, target_cycle));
738                 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1);
739         }
740
741         /* If we get non-NULL, loop */
742         jit_patch_at(jit_bnei(JIT_R0, 0), loop);
743
744         to_end2 = jit_jmpi();
745
746         /* When exiting, the recompiled code will jump to that address */
747         jit_note(__FILE__, __LINE__);
748         jit_patch(to_end);
749
750         /* Store back the next_pc to the lightrec_state structure */
751         offset = offsetof(struct lightrec_state, next_pc);
752         jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0);
753
754         jit_patch(to_end2);
755
756         jit_retr(LIGHTREC_REG_CYCLE);
757         jit_epilog();
758
759         block->state = state;
760         block->_jit = _jit;
761         block->function = jit_emit();
762         block->opcode_list = NULL;
763         block->flags = 0;
764         block->nb_ops = 0;
765
766         jit_get_code(&code_size);
767         lightrec_register(MEM_FOR_CODE, code_size);
768
769         block->code_size = code_size;
770
771         state->eob_wrapper_func = jit_address(addr2);
772         state->get_next_block = jit_address(addr);
773
774         if (ENABLE_DISASSEMBLER) {
775                 pr_debug("Dispatcher block:\n");
776                 jit_disassemble();
777         }
778
779         /* We're done! */
780         jit_clear_state();
781         return block;
782
783 err_free_block:
784         lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
785 err_no_mem:
786         pr_err("Unable to compile dispatcher: Out of memory\n");
787         return NULL;
788 }
789
790 union code lightrec_read_opcode(struct lightrec_state *state, u32 pc)
791 {
792         u32 addr, kunseg_pc = kunseg(pc);
793         const u32 *code;
794         const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
795
796         addr = kunseg_pc - map->pc;
797
798         while (map->mirror_of)
799                 map = map->mirror_of;
800
801         code = map->address + addr;
802
803         return (union code) *code;
804 }
805
806 static struct block * lightrec_precompile_block(struct lightrec_state *state,
807                                                 u32 pc)
808 {
809         struct opcode *list;
810         struct block *block;
811         const u32 *code;
812         u32 addr, kunseg_pc = kunseg(pc);
813         const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
814         unsigned int length;
815
816         if (!map)
817                 return NULL;
818
819         addr = kunseg_pc - map->pc;
820
821         while (map->mirror_of)
822                 map = map->mirror_of;
823
824         code = map->address + addr;
825
826         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
827         if (!block) {
828                 pr_err("Unable to recompile block: Out of memory\n");
829                 return NULL;
830         }
831
832         list = lightrec_disassemble(state, code, &length);
833         if (!list) {
834                 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
835                 return NULL;
836         }
837
838         block->pc = pc;
839         block->state = state;
840         block->_jit = NULL;
841         block->function = NULL;
842         block->opcode_list = list;
843         block->map = map;
844         block->next = NULL;
845         block->flags = 0;
846         block->code_size = 0;
847 #if ENABLE_THREADED_COMPILER
848         block->op_list_freed = (atomic_flag)ATOMIC_FLAG_INIT;
849 #endif
850         block->nb_ops = length / sizeof(u32);
851
852         lightrec_optimize(block);
853
854         length = block->nb_ops * sizeof(u32);
855
856         lightrec_register(MEM_FOR_MIPS_CODE, length);
857
858         if (ENABLE_DISASSEMBLER) {
859                 pr_debug("Disassembled block at PC: 0x%x\n", block->pc);
860                 lightrec_print_disassembly(block, code, length);
861         }
862
863         pr_debug("Block size: %lu opcodes\n", block->nb_ops);
864
865         /* If the first opcode is an 'impossible' branch, never compile the
866          * block */
867         if (list->flags & LIGHTREC_EMULATE_BRANCH)
868                 block->flags |= BLOCK_NEVER_COMPILE;
869
870         block->hash = lightrec_calculate_block_hash(block);
871
872         pr_debug("Recompile count: %u\n", state->nb_precompile++);
873
874         return block;
875 }
876
877 static bool lightrec_block_is_fully_tagged(struct block *block)
878 {
879         struct opcode *op;
880
881         for (op = block->opcode_list; op; op = op->next) {
882                 /* Verify that all load/stores of the opcode list
883                  * Check all loads/stores of the opcode list and mark the
884                  * block as fully compiled if they all have been tagged. */
885                 switch (op->c.i.op) {
886                 case OP_LB:
887                 case OP_LH:
888                 case OP_LWL:
889                 case OP_LW:
890                 case OP_LBU:
891                 case OP_LHU:
892                 case OP_LWR:
893                 case OP_SB:
894                 case OP_SH:
895                 case OP_SWL:
896                 case OP_SW:
897                 case OP_SWR:
898                 case OP_LWC2:
899                 case OP_SWC2:
900                         if (!(op->flags & (LIGHTREC_DIRECT_IO |
901                                            LIGHTREC_HW_IO)))
902                                 return false;
903                 default: /* fall-through */
904                         continue;
905                 }
906         }
907
908         return true;
909 }
910
911 static void lightrec_reap_block(void *data)
912 {
913         struct block *block = data;
914
915         pr_debug("Reap dead block at PC 0x%08x\n", block->pc);
916         lightrec_free_block(block);
917 }
918
919 static void lightrec_reap_jit(void *data)
920 {
921         _jit_destroy_state(data);
922 }
923
924 int lightrec_compile_block(struct block *block)
925 {
926         struct lightrec_state *state = block->state;
927         struct lightrec_branch_target *target;
928         bool op_list_freed = false, fully_tagged = false;
929         struct block *block2;
930         struct opcode *elm;
931         jit_state_t *_jit, *oldjit;
932         jit_node_t *start_of_block;
933         bool skip_next = false;
934         jit_word_t code_size;
935         unsigned int i, j;
936         u32 next_pc, offset;
937
938         fully_tagged = lightrec_block_is_fully_tagged(block);
939         if (fully_tagged)
940                 block->flags |= BLOCK_FULLY_TAGGED;
941
942         _jit = jit_new_state();
943         if (!_jit)
944                 return -ENOMEM;
945
946         oldjit = block->_jit;
947         block->_jit = _jit;
948
949         lightrec_regcache_reset(state->reg_cache);
950         state->cycles = 0;
951         state->nb_branches = 0;
952         state->nb_local_branches = 0;
953         state->nb_targets = 0;
954
955         jit_prolog();
956         jit_tramp(256);
957
958         start_of_block = jit_label();
959
960         for (elm = block->opcode_list; elm; elm = elm->next) {
961                 next_pc = block->pc + elm->offset * sizeof(u32);
962
963                 if (skip_next) {
964                         skip_next = false;
965                         continue;
966                 }
967
968                 state->cycles += lightrec_cycles_of_opcode(elm->c);
969
970                 if (elm->flags & LIGHTREC_EMULATE_BRANCH) {
971                         pr_debug("Branch at offset 0x%x will be emulated\n",
972                                  elm->offset << 2);
973                         lightrec_emit_eob(block, elm, next_pc);
974                         skip_next = !(elm->flags & LIGHTREC_NO_DS);
975                 } else if (elm->opcode) {
976                         lightrec_rec_opcode(block, elm, next_pc);
977                         skip_next = has_delay_slot(elm->c) &&
978                                 !(elm->flags & LIGHTREC_NO_DS);
979 #if _WIN32
980                         /* FIXME: GNU Lightning on Windows seems to use our
981                          * mapped registers as temporaries. Until the actual bug
982                          * is found and fixed, unconditionally mark our
983                          * registers as live here. */
984                         lightrec_regcache_mark_live(state->reg_cache, _jit);
985 #endif
986                 }
987         }
988
989         for (i = 0; i < state->nb_branches; i++)
990                 jit_patch(state->branches[i]);
991
992         for (i = 0; i < state->nb_local_branches; i++) {
993                 struct lightrec_branch *branch = &state->local_branches[i];
994
995                 pr_debug("Patch local branch to offset 0x%x\n",
996                          branch->target << 2);
997
998                 if (branch->target == 0) {
999                         jit_patch_at(branch->branch, start_of_block);
1000                         continue;
1001                 }
1002
1003                 for (j = 0; j < state->nb_targets; j++) {
1004                         if (state->targets[j].offset == branch->target) {
1005                                 jit_patch_at(branch->branch,
1006                                              state->targets[j].label);
1007                                 break;
1008                         }
1009                 }
1010
1011                 if (j == state->nb_targets)
1012                         pr_err("Unable to find branch target\n");
1013         }
1014
1015         jit_ldxi(JIT_R0, LIGHTREC_REG_STATE,
1016                  offsetof(struct lightrec_state, eob_wrapper_func));
1017
1018         jit_jmpr(JIT_R0);
1019
1020         jit_ret();
1021         jit_epilog();
1022
1023         block->function = jit_emit();
1024         block->flags &= ~BLOCK_SHOULD_RECOMPILE;
1025
1026         /* Add compiled function to the LUT */
1027         state->code_lut[lut_offset(block->pc)] = block->function;
1028
1029         /* Fill code LUT with the block's entry points */
1030         for (i = 0; i < state->nb_targets; i++) {
1031                 target = &state->targets[i];
1032
1033                 if (target->offset) {
1034                         offset = lut_offset(block->pc) + target->offset;
1035                         state->code_lut[offset] = jit_address(target->label);
1036                 }
1037         }
1038
1039         /* Detect old blocks that have been covered by the new one */
1040         for (i = 0; i < state->nb_targets; i++) {
1041                 target = &state->targets[i];
1042
1043                 if (!target->offset)
1044                         continue;
1045
1046                 offset = block->pc + target->offset * sizeof(u32);
1047                 block2 = lightrec_find_block(state->block_cache, offset);
1048                 if (block2) {
1049                         /* No need to check if block2 is compilable - it must
1050                          * be, otherwise block wouldn't be compilable either */
1051
1052                         block2->flags |= BLOCK_IS_DEAD;
1053
1054                         pr_debug("Reap block 0x%08x as it's covered by block "
1055                                  "0x%08x\n", block2->pc, block->pc);
1056
1057                         lightrec_unregister_block(state->block_cache, block2);
1058
1059                         if (ENABLE_THREADED_COMPILER) {
1060                                 lightrec_recompiler_remove(state->rec, block2);
1061                                 lightrec_reaper_add(state->reaper,
1062                                                     lightrec_reap_block,
1063                                                     block2);
1064                         } else {
1065                                 lightrec_free_block(block2);
1066                         }
1067                 }
1068         }
1069
1070         jit_get_code(&code_size);
1071         lightrec_register(MEM_FOR_CODE, code_size);
1072
1073         block->code_size = code_size;
1074
1075         if (ENABLE_DISASSEMBLER) {
1076                 pr_debug("Compiling block at PC: 0x%x\n", block->pc);
1077                 jit_disassemble();
1078         }
1079
1080         jit_clear_state();
1081
1082 #if ENABLE_THREADED_COMPILER
1083         if (fully_tagged)
1084                 op_list_freed = atomic_flag_test_and_set(&block->op_list_freed);
1085 #endif
1086         if (fully_tagged && !op_list_freed) {
1087                 pr_debug("Block PC 0x%08x is fully tagged"
1088                          " - free opcode list\n", block->pc);
1089                 lightrec_free_opcode_list(state, block->opcode_list);
1090                 block->opcode_list = NULL;
1091         }
1092
1093         if (oldjit) {
1094                 pr_debug("Block 0x%08x recompiled, reaping old jit context.\n",
1095                          block->pc);
1096
1097                 if (ENABLE_THREADED_COMPILER)
1098                         lightrec_reaper_add(state->reaper,
1099                                             lightrec_reap_jit, oldjit);
1100                 else
1101                         _jit_destroy_state(oldjit);
1102         }
1103
1104         return 0;
1105 }
1106
1107 u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle)
1108 {
1109         s32 (*func)(void *, s32) = (void *)state->dispatcher->function;
1110         void *block_trace;
1111         s32 cycles_delta;
1112
1113         state->exit_flags = LIGHTREC_EXIT_NORMAL;
1114
1115         /* Handle the cycle counter overflowing */
1116         if (unlikely(target_cycle < state->current_cycle))
1117                 target_cycle = UINT_MAX;
1118
1119         state->target_cycle = target_cycle;
1120
1121         block_trace = get_next_block_func(state, pc);
1122         if (block_trace) {
1123                 cycles_delta = state->target_cycle - state->current_cycle;
1124
1125                 cycles_delta = (*func)(block_trace, cycles_delta);
1126
1127                 state->current_cycle = state->target_cycle - cycles_delta;
1128         }
1129
1130         if (ENABLE_THREADED_COMPILER)
1131                 lightrec_reaper_reap(state->reaper);
1132
1133         return state->next_pc;
1134 }
1135
1136 u32 lightrec_execute_one(struct lightrec_state *state, u32 pc)
1137 {
1138         return lightrec_execute(state, pc, state->current_cycle);
1139 }
1140
1141 u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc)
1142 {
1143         struct block *block = lightrec_get_block(state, pc);
1144         if (!block)
1145                 return 0;
1146
1147         state->exit_flags = LIGHTREC_EXIT_NORMAL;
1148
1149         return lightrec_emulate_block(block, pc);
1150 }
1151
1152 void lightrec_free_block(struct block *block)
1153 {
1154         lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32));
1155         if (block->opcode_list)
1156                 lightrec_free_opcode_list(block->state, block->opcode_list);
1157         if (block->_jit)
1158                 _jit_destroy_state(block->_jit);
1159         lightrec_unregister(MEM_FOR_CODE, block->code_size);
1160         lightrec_free(block->state, MEM_FOR_IR, sizeof(*block), block);
1161 }
1162
1163 struct lightrec_state * lightrec_init(char *argv0,
1164                                       const struct lightrec_mem_map *map,
1165                                       size_t nb,
1166                                       const struct lightrec_ops *ops)
1167 {
1168         struct lightrec_state *state;
1169
1170         /* Sanity-check ops */
1171         if (!ops ||
1172             !ops->cop0_ops.mfc || !ops->cop0_ops.cfc || !ops->cop0_ops.mtc ||
1173             !ops->cop0_ops.ctc || !ops->cop0_ops.op ||
1174             !ops->cop2_ops.mfc || !ops->cop2_ops.cfc || !ops->cop2_ops.mtc ||
1175             !ops->cop2_ops.ctc || !ops->cop2_ops.op) {
1176                 pr_err("Missing callbacks in lightrec_ops structure\n");
1177                 return NULL;
1178         }
1179
1180         init_jit(argv0);
1181
1182         state = calloc(1, sizeof(*state) +
1183                        sizeof(*state->code_lut) * CODE_LUT_SIZE);
1184         if (!state)
1185                 goto err_finish_jit;
1186
1187         lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) +
1188                           sizeof(*state->code_lut) * CODE_LUT_SIZE);
1189
1190 #if ENABLE_TINYMM
1191         state->tinymm = tinymm_init(malloc, free, 4096);
1192         if (!state->tinymm)
1193                 goto err_free_state;
1194 #endif
1195
1196         state->block_cache = lightrec_blockcache_init(state);
1197         if (!state->block_cache)
1198                 goto err_free_tinymm;
1199
1200         state->reg_cache = lightrec_regcache_init(state);
1201         if (!state->reg_cache)
1202                 goto err_free_block_cache;
1203
1204         if (ENABLE_THREADED_COMPILER) {
1205                 state->rec = lightrec_recompiler_init(state);
1206                 if (!state->rec)
1207                         goto err_free_reg_cache;
1208
1209                 state->reaper = lightrec_reaper_init(state);
1210                 if (!state->reaper)
1211                         goto err_free_recompiler;
1212         }
1213
1214         state->nb_maps = nb;
1215         state->maps = map;
1216
1217         memcpy(&state->ops, ops, sizeof(*ops));
1218
1219         state->dispatcher = generate_dispatcher(state);
1220         if (!state->dispatcher)
1221                 goto err_free_reaper;
1222
1223         state->rw_generic_wrapper = generate_wrapper(state,
1224                                                      lightrec_rw_generic_cb,
1225                                                      true);
1226         if (!state->rw_generic_wrapper)
1227                 goto err_free_dispatcher;
1228
1229         state->rw_wrapper = generate_wrapper(state, lightrec_rw_cb, false);
1230         if (!state->rw_wrapper)
1231                 goto err_free_generic_rw_wrapper;
1232
1233         state->mfc_wrapper = generate_wrapper(state, lightrec_mfc_cb, false);
1234         if (!state->mfc_wrapper)
1235                 goto err_free_rw_wrapper;
1236
1237         state->mtc_wrapper = generate_wrapper(state, lightrec_mtc_cb, false);
1238         if (!state->mtc_wrapper)
1239                 goto err_free_mfc_wrapper;
1240
1241         state->rfe_wrapper = generate_wrapper(state, lightrec_rfe_cb, false);
1242         if (!state->rfe_wrapper)
1243                 goto err_free_mtc_wrapper;
1244
1245         state->cp_wrapper = generate_wrapper(state, lightrec_cp_cb, false);
1246         if (!state->cp_wrapper)
1247                 goto err_free_rfe_wrapper;
1248
1249         state->syscall_wrapper = generate_wrapper(state, lightrec_syscall_cb,
1250                                                   false);
1251         if (!state->syscall_wrapper)
1252                 goto err_free_cp_wrapper;
1253
1254         state->break_wrapper = generate_wrapper(state, lightrec_break_cb,
1255                                                 false);
1256         if (!state->break_wrapper)
1257                 goto err_free_syscall_wrapper;
1258
1259         state->rw_generic_func = state->rw_generic_wrapper->function;
1260         state->rw_func = state->rw_wrapper->function;
1261         state->mfc_func = state->mfc_wrapper->function;
1262         state->mtc_func = state->mtc_wrapper->function;
1263         state->rfe_func = state->rfe_wrapper->function;
1264         state->cp_func = state->cp_wrapper->function;
1265         state->syscall_func = state->syscall_wrapper->function;
1266         state->break_func = state->break_wrapper->function;
1267
1268         map = &state->maps[PSX_MAP_BIOS];
1269         state->offset_bios = (uintptr_t)map->address - map->pc;
1270
1271         map = &state->maps[PSX_MAP_SCRATCH_PAD];
1272         state->offset_scratch = (uintptr_t)map->address - map->pc;
1273
1274         map = &state->maps[PSX_MAP_KERNEL_USER_RAM];
1275         state->offset_ram = (uintptr_t)map->address - map->pc;
1276
1277         if (state->maps[PSX_MAP_MIRROR1].address == map->address + 0x200000 &&
1278             state->maps[PSX_MAP_MIRROR2].address == map->address + 0x400000 &&
1279             state->maps[PSX_MAP_MIRROR3].address == map->address + 0x600000)
1280                 state->mirrors_mapped = true;
1281
1282         return state;
1283
1284 err_free_syscall_wrapper:
1285         lightrec_free_block(state->syscall_wrapper);
1286 err_free_cp_wrapper:
1287         lightrec_free_block(state->cp_wrapper);
1288 err_free_rfe_wrapper:
1289         lightrec_free_block(state->rfe_wrapper);
1290 err_free_mtc_wrapper:
1291         lightrec_free_block(state->mtc_wrapper);
1292 err_free_mfc_wrapper:
1293         lightrec_free_block(state->mfc_wrapper);
1294 err_free_rw_wrapper:
1295         lightrec_free_block(state->rw_wrapper);
1296 err_free_generic_rw_wrapper:
1297         lightrec_free_block(state->rw_generic_wrapper);
1298 err_free_dispatcher:
1299         lightrec_free_block(state->dispatcher);
1300 err_free_reaper:
1301         if (ENABLE_THREADED_COMPILER)
1302                 lightrec_reaper_destroy(state->reaper);
1303 err_free_recompiler:
1304         if (ENABLE_THREADED_COMPILER)
1305                 lightrec_free_recompiler(state->rec);
1306 err_free_reg_cache:
1307         lightrec_free_regcache(state->reg_cache);
1308 err_free_block_cache:
1309         lightrec_free_block_cache(state->block_cache);
1310 err_free_tinymm:
1311 #if ENABLE_TINYMM
1312         tinymm_shutdown(state->tinymm);
1313 err_free_state:
1314 #endif
1315         lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1316                             sizeof(*state->code_lut) * CODE_LUT_SIZE);
1317         free(state);
1318 err_finish_jit:
1319         finish_jit();
1320         return NULL;
1321 }
1322
1323 void lightrec_destroy(struct lightrec_state *state)
1324 {
1325         if (ENABLE_THREADED_COMPILER) {
1326                 lightrec_free_recompiler(state->rec);
1327                 lightrec_reaper_destroy(state->reaper);
1328         }
1329
1330         lightrec_free_regcache(state->reg_cache);
1331         lightrec_free_block_cache(state->block_cache);
1332         lightrec_free_block(state->dispatcher);
1333         lightrec_free_block(state->rw_generic_wrapper);
1334         lightrec_free_block(state->rw_wrapper);
1335         lightrec_free_block(state->mfc_wrapper);
1336         lightrec_free_block(state->mtc_wrapper);
1337         lightrec_free_block(state->rfe_wrapper);
1338         lightrec_free_block(state->cp_wrapper);
1339         lightrec_free_block(state->syscall_wrapper);
1340         lightrec_free_block(state->break_wrapper);
1341         finish_jit();
1342
1343 #if ENABLE_TINYMM
1344         tinymm_shutdown(state->tinymm);
1345 #endif
1346         lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1347                             sizeof(*state->code_lut) * CODE_LUT_SIZE);
1348         free(state);
1349 }
1350
1351 void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len)
1352 {
1353         u32 kaddr = kunseg(addr & ~0x3);
1354         const struct lightrec_mem_map *map = lightrec_get_map(state, kaddr);
1355
1356         if (map) {
1357                 while (map->mirror_of)
1358                         map = map->mirror_of;
1359
1360                 if (map != &state->maps[PSX_MAP_KERNEL_USER_RAM])
1361                         return;
1362
1363                 /* Handle mirrors */
1364                 kaddr &= (state->maps[PSX_MAP_KERNEL_USER_RAM].length - 1);
1365
1366                 for (; len > 4; len -= 4, kaddr += 4)
1367                         lightrec_invalidate_map(state, map, kaddr);
1368
1369                 lightrec_invalidate_map(state, map, kaddr);
1370         }
1371 }
1372
1373 void lightrec_invalidate_all(struct lightrec_state *state)
1374 {
1375         memset(state->code_lut, 0, sizeof(*state->code_lut) * CODE_LUT_SIZE);
1376 }
1377
1378 void lightrec_set_invalidate_mode(struct lightrec_state *state, bool dma_only)
1379 {
1380         if (state->invalidate_from_dma_only != dma_only)
1381                 lightrec_invalidate_all(state);
1382
1383         state->invalidate_from_dma_only = dma_only;
1384 }
1385
1386 void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags)
1387 {
1388         if (flags != LIGHTREC_EXIT_NORMAL) {
1389                 state->exit_flags |= flags;
1390                 state->target_cycle = state->current_cycle;
1391         }
1392 }
1393
1394 u32 lightrec_exit_flags(struct lightrec_state *state)
1395 {
1396         return state->exit_flags;
1397 }
1398
1399 void lightrec_dump_registers(struct lightrec_state *state, u32 regs[34])
1400 {
1401         memcpy(regs, state->native_reg_cache, sizeof(state->native_reg_cache));
1402 }
1403
1404 void lightrec_restore_registers(struct lightrec_state *state, u32 regs[34])
1405 {
1406         memcpy(state->native_reg_cache, regs, sizeof(state->native_reg_cache));
1407 }
1408
1409 u32 lightrec_current_cycle_count(const struct lightrec_state *state)
1410 {
1411         return state->current_cycle;
1412 }
1413
1414 void lightrec_reset_cycle_count(struct lightrec_state *state, u32 cycles)
1415 {
1416         state->current_cycle = cycles;
1417
1418         if (state->target_cycle < cycles)
1419                 state->target_cycle = cycles;
1420 }
1421
1422 void lightrec_set_target_cycle_count(struct lightrec_state *state, u32 cycles)
1423 {
1424         if (state->exit_flags == LIGHTREC_EXIT_NORMAL) {
1425                 if (cycles < state->current_cycle)
1426                         cycles = state->current_cycle;
1427
1428                 state->target_cycle = cycles;
1429         }
1430 }