deps/lightrec/lightrec.c

   1 /*
   2  * Copyright (C) 2014-2020 Paul Cercueil <paul@crapouillou.net>
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  */
  14
  15 #include "blockcache.h"
  16 #include "config.h"
  17 #include "debug.h"
  18 #include "disassembler.h"
  19 #include "emitter.h"
  20 #include "interpreter.h"
  21 #include "lightrec.h"
  22 #include "memmanager.h"
  23 #include "reaper.h"
  24 #include "recompiler.h"
  25 #include "regcache.h"
  26 #include "optimizer.h"
  27
  28 #include <errno.h>
  29 #include <lightning.h>
  30 #include <limits.h>
  31 #if ENABLE_THREADED_COMPILER
  32 #include <stdatomic.h>
  33 #endif
  34 #include <stdbool.h>
  35 #include <stddef.h>
  36 #include <string.h>
  37 #if ENABLE_TINYMM
  38 #include <tinymm.h>
  39 #endif
  40
  41 #define GENMASK(h, l) \
  42         (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
  43
  44 static struct block * lightrec_precompile_block(struct lightrec_state *state,
  45                                                 u32 pc);
  46
  47 static void lightrec_default_sb(struct lightrec_state *state, u32 opcode,
  48                                 void *host, u32 addr, u8 data)
  49 {
  50         *(u8 *)host = data;
  51
  52         if (!state->invalidate_from_dma_only)
  53                 lightrec_invalidate(state, addr, 1);
  54 }
  55
  56 static void lightrec_default_sh(struct lightrec_state *state, u32 opcode,
  57                                 void *host, u32 addr, u16 data)
  58 {
  59         *(u16 *)host = HTOLE16(data);
  60
  61         if (!state->invalidate_from_dma_only)
  62                 lightrec_invalidate(state, addr, 2);
  63 }
  64
  65 static void lightrec_default_sw(struct lightrec_state *state, u32 opcode,
  66                                 void *host, u32 addr, u32 data)
  67 {
  68         *(u32 *)host = HTOLE32(data);
  69
  70         if (!state->invalidate_from_dma_only)
  71                 lightrec_invalidate(state, addr, 4);
  72 }
  73
  74 static u8 lightrec_default_lb(struct lightrec_state *state,
  75                               u32 opcode, void *host, u32 addr)
  76 {
  77         return *(u8 *)host;
  78 }
  79
  80 static u16 lightrec_default_lh(struct lightrec_state *state,
  81                                u32 opcode, void *host, u32 addr)
  82 {
  83         return LE16TOH(*(u16 *)host);
  84 }
  85
  86 static u32 lightrec_default_lw(struct lightrec_state *state,
  87                                u32 opcode, void *host, u32 addr)
  88 {
  89         return LE32TOH(*(u32 *)host);
  90 }
  91
  92 static const struct lightrec_mem_map_ops lightrec_default_ops = {
  93         .sb = lightrec_default_sb,
  94         .sh = lightrec_default_sh,
  95         .sw = lightrec_default_sw,
  96         .lb = lightrec_default_lb,
  97         .lh = lightrec_default_lh,
  98         .lw = lightrec_default_lw,
  99 };
 100
 101 static void __segfault_cb(struct lightrec_state *state, u32 addr)
 102 {
 103         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
 104         pr_err("Segmentation fault in recompiled code: invalid "
 105                "load/store at address 0x%08x\n", addr);
 106 }
 107
 108 static void lightrec_swl(struct lightrec_state *state,
 109                          const struct lightrec_mem_map_ops *ops,
 110                          u32 opcode, void *host, u32 addr, u32 data)
 111 {
 112         unsigned int shift = addr & 0x3;
 113         unsigned int mask = GENMASK(31, (shift + 1) * 8);
 114         u32 old_data;
 115
 116         /* Align to 32 bits */
 117         addr &= ~3;
 118         host = (void *)((uintptr_t)host & ~3);
 119
 120         old_data = ops->lw(state, opcode, host, addr);
 121
 122         data = (data >> ((3 - shift) * 8)) | (old_data & mask);
 123
 124         ops->sw(state, opcode, host, addr, data);
 125 }
 126
 127 static void lightrec_swr(struct lightrec_state *state,
 128                          const struct lightrec_mem_map_ops *ops,
 129                          u32 opcode, void *host, u32 addr, u32 data)
 130 {
 131         unsigned int shift = addr & 0x3;
 132         unsigned int mask = (1 << (shift * 8)) - 1;
 133         u32 old_data;
 134
 135         /* Align to 32 bits */
 136         addr &= ~3;
 137         host = (void *)((uintptr_t)host & ~3);
 138
 139         old_data = ops->lw(state, opcode, host, addr);
 140
 141         data = (data << (shift * 8)) | (old_data & mask);
 142
 143         ops->sw(state, opcode, host, addr, data);
 144 }
 145
 146 static void lightrec_swc2(struct lightrec_state *state, union code op,
 147                           const struct lightrec_mem_map_ops *ops,
 148                           void *host, u32 addr)
 149 {
 150         u32 data = state->ops.cop2_ops.mfc(state, op.opcode, op.i.rt);
 151
 152         ops->sw(state, op.opcode, host, addr, data);
 153 }
 154
 155 static u32 lightrec_lwl(struct lightrec_state *state,
 156                         const struct lightrec_mem_map_ops *ops,
 157                         u32 opcode, void *host, u32 addr, u32 data)
 158 {
 159         unsigned int shift = addr & 0x3;
 160         unsigned int mask = (1 << (24 - shift * 8)) - 1;
 161         u32 old_data;
 162
 163         /* Align to 32 bits */
 164         addr &= ~3;
 165         host = (void *)((uintptr_t)host & ~3);
 166
 167         old_data = ops->lw(state, opcode, host, addr);
 168
 169         return (data & mask) | (old_data << (24 - shift * 8));
 170 }
 171
 172 static u32 lightrec_lwr(struct lightrec_state *state,
 173                         const struct lightrec_mem_map_ops *ops,
 174                         u32 opcode, void *host, u32 addr, u32 data)
 175 {
 176         unsigned int shift = addr & 0x3;
 177         unsigned int mask = GENMASK(31, 32 - shift * 8);
 178         u32 old_data;
 179
 180         /* Align to 32 bits */
 181         addr &= ~3;
 182         host = (void *)((uintptr_t)host & ~3);
 183
 184         old_data = ops->lw(state, opcode, host, addr);
 185
 186         return (data & mask) | (old_data >> (shift * 8));
 187 }
 188
 189 static void lightrec_lwc2(struct lightrec_state *state, union code op,
 190                           const struct lightrec_mem_map_ops *ops,
 191                           void *host, u32 addr)
 192 {
 193         u32 data = ops->lw(state, op.opcode, host, addr);
 194
 195         state->ops.cop2_ops.mtc(state, op.opcode, op.i.rt, data);
 196 }
 197
 198 static void lightrec_invalidate_map(struct lightrec_state *state,
 199                 const struct lightrec_mem_map *map, u32 addr)
 200 {
 201         if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM])
 202                 state->code_lut[lut_offset(addr)] = NULL;
 203 }
 204
 205 static const struct lightrec_mem_map *
 206 lightrec_get_map(struct lightrec_state *state, u32 kaddr)
 207 {
 208         unsigned int i;
 209
 210         for (i = 0; i < state->nb_maps; i++) {
 211                 const struct lightrec_mem_map *map = &state->maps[i];
 212
 213                 if (kaddr >= map->pc && kaddr < map->pc + map->length)
 214                         return map;
 215         }
 216
 217         return NULL;
 218 }
 219
 220 u32 lightrec_rw(struct lightrec_state *state, union code op,
 221                 u32 addr, u32 data, u16 *flags)
 222 {
 223         const struct lightrec_mem_map *map;
 224         const struct lightrec_mem_map_ops *ops;
 225         u32 kaddr, pc, opcode = op.opcode;
 226         void *host;
 227
 228         addr += (s16) op.i.imm;
 229         kaddr = kunseg(addr);
 230
 231         map = lightrec_get_map(state, kaddr);
 232         if (!map) {
 233                 __segfault_cb(state, addr);
 234                 return 0;
 235         }
 236
 237         pc = map->pc;
 238
 239         while (map->mirror_of)
 240                 map = map->mirror_of;
 241
 242         host = (void *)((uintptr_t)map->address + kaddr - pc);
 243
 244         if (unlikely(map->ops)) {
 245                 if (flags)
 246                         *flags |= LIGHTREC_HW_IO;
 247
 248                 ops = map->ops;
 249         } else {
 250                 if (flags)
 251                         *flags |= LIGHTREC_DIRECT_IO;
 252
 253                 ops = &lightrec_default_ops;
 254         }
 255
 256         switch (op.i.op) {
 257         case OP_SB:
 258                 ops->sb(state, opcode, host, addr, (u8) data);
 259                 return 0;
 260         case OP_SH:
 261                 ops->sh(state, opcode, host, addr, (u16) data);
 262                 return 0;
 263         case OP_SWL:
 264                 lightrec_swl(state, ops, opcode, host, addr, data);
 265                 return 0;
 266         case OP_SWR:
 267                 lightrec_swr(state, ops, opcode, host, addr, data);
 268                 return 0;
 269         case OP_SW:
 270                 ops->sw(state, opcode, host, addr, data);
 271                 return 0;
 272         case OP_SWC2:
 273                 lightrec_swc2(state, op, ops, host, addr);
 274                 return 0;
 275         case OP_LB:
 276                 return (s32) (s8) ops->lb(state, opcode, host, addr);
 277         case OP_LBU:
 278                 return ops->lb(state, opcode, host, addr);
 279         case OP_LH:
 280                 return (s32) (s16) ops->lh(state, opcode, host, addr);
 281         case OP_LHU:
 282                 return ops->lh(state, opcode, host, addr);
 283         case OP_LWC2:
 284                 lightrec_lwc2(state, op, ops, host, addr);
 285                 return 0;
 286         case OP_LWL:
 287                 return lightrec_lwl(state, ops, opcode, host, addr, data);
 288         case OP_LWR:
 289                 return lightrec_lwr(state, ops, opcode, host, addr, data);
 290         case OP_LW:
 291         default:
 292                 return ops->lw(state, opcode, host, addr);
 293         }
 294 }
 295
 296 static void lightrec_rw_helper(struct lightrec_state *state,
 297                                union code op, u16 *flags)
 298 {
 299         u32 ret = lightrec_rw(state, op,
 300                           state->native_reg_cache[op.i.rs],
 301                           state->native_reg_cache[op.i.rt], flags);
 302
 303         switch (op.i.op) {
 304         case OP_LB:
 305         case OP_LBU:
 306         case OP_LH:
 307         case OP_LHU:
 308         case OP_LWL:
 309         case OP_LWR:
 310         case OP_LW:
 311                 if (op.i.rt)
 312                         state->native_reg_cache[op.i.rt] = ret;
 313         default: /* fall-through */
 314                 break;
 315         }
 316 }
 317
 318 static void lightrec_rw_cb(struct lightrec_state *state, union code op)
 319 {
 320         lightrec_rw_helper(state, op, NULL);
 321 }
 322
 323 static void lightrec_rw_generic_cb(struct lightrec_state *state,
 324                                    struct opcode *op, struct block *block)
 325 {
 326         bool was_tagged = op->flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
 327
 328         lightrec_rw_helper(state, op->c, &op->flags);
 329
 330         if (!was_tagged) {
 331                 pr_debug("Opcode of block at PC 0x%08x offset 0x%x has been "
 332                          "tagged - flag for recompilation\n",
 333                          block->pc, op->offset << 2);
 334
 335                 block->flags |= BLOCK_SHOULD_RECOMPILE;
 336         }
 337 }
 338
 339 u32 lightrec_mfc(struct lightrec_state *state, union code op)
 340 {
 341         bool is_cfc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CFC0) ||
 342                       (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CFC2);
 343         u32 (*func)(struct lightrec_state *, u32, u8);
 344         const struct lightrec_cop_ops *ops;
 345
 346         if (op.i.op == OP_CP0)
 347                 ops = &state->ops.cop0_ops;
 348         else
 349                 ops = &state->ops.cop2_ops;
 350
 351         if (is_cfc)
 352                 func = ops->cfc;
 353         else
 354                 func = ops->mfc;
 355
 356         return (*func)(state, op.opcode, op.r.rd);
 357 }
 358
 359 static void lightrec_mfc_cb(struct lightrec_state *state, union code op)
 360 {
 361         u32 rt = lightrec_mfc(state, op);
 362
 363         if (op.r.rt)
 364                 state->native_reg_cache[op.r.rt] = rt;
 365 }
 366
 367 void lightrec_mtc(struct lightrec_state *state, union code op, u32 data)
 368 {
 369         bool is_ctc = (op.i.op == OP_CP0 && op.r.rs == OP_CP0_CTC0) ||
 370                       (op.i.op == OP_CP2 && op.r.rs == OP_CP2_BASIC_CTC2);
 371         void (*func)(struct lightrec_state *, u32, u8, u32);
 372         const struct lightrec_cop_ops *ops;
 373
 374         if (op.i.op == OP_CP0)
 375                 ops = &state->ops.cop0_ops;
 376         else
 377                 ops = &state->ops.cop2_ops;
 378
 379         if (is_ctc)
 380                 func = ops->ctc;
 381         else
 382                 func = ops->mtc;
 383
 384         (*func)(state, op.opcode, op.r.rd, data);
 385 }
 386
 387 static void lightrec_mtc_cb(struct lightrec_state *state, union code op)
 388 {
 389         lightrec_mtc(state, op, state->native_reg_cache[op.r.rt]);
 390 }
 391
 392 static void lightrec_rfe_cb(struct lightrec_state *state, union code op)
 393 {
 394         u32 status;
 395
 396         /* Read CP0 Status register (r12) */
 397         status = state->ops.cop0_ops.mfc(state, op.opcode, 12);
 398
 399         /* Switch the bits */
 400         status = ((status & 0x3c) >> 2) | (status & ~0xf);
 401
 402         /* Write it back */
 403         state->ops.cop0_ops.ctc(state, op.opcode, 12, status);
 404 }
 405
 406 static void lightrec_cp_cb(struct lightrec_state *state, union code op)
 407 {
 408         void (*func)(struct lightrec_state *, u32);
 409
 410         if ((op.opcode >> 25) & 1)
 411                 func = state->ops.cop2_ops.op;
 412         else
 413                 func = state->ops.cop0_ops.op;
 414
 415         (*func)(state, op.opcode);
 416 }
 417
 418 static void lightrec_syscall_cb(struct lightrec_state *state, union code op)
 419 {
 420         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SYSCALL);
 421 }
 422
 423 static void lightrec_break_cb(struct lightrec_state *state, union code op)
 424 {
 425         lightrec_set_exit_flags(state, LIGHTREC_EXIT_BREAK);
 426 }
 427
 428 struct block * lightrec_get_block(struct lightrec_state *state, u32 pc)
 429 {
 430         struct block *block = lightrec_find_block(state->block_cache, pc);
 431
 432         if (block && lightrec_block_is_outdated(block)) {
 433                 pr_debug("Block at PC 0x%08x is outdated!\n", block->pc);
 434
 435                 /* Make sure the recompiler isn't processing the block we'll
 436                  * destroy */
 437                 if (ENABLE_THREADED_COMPILER)
 438                         lightrec_recompiler_remove(state->rec, block);
 439
 440                 lightrec_unregister_block(state->block_cache, block);
 441                 remove_from_code_lut(state->block_cache, block);
 442                 lightrec_free_block(block);
 443                 block = NULL;
 444         }
 445
 446         if (!block) {
 447                 block = lightrec_precompile_block(state, pc);
 448                 if (!block) {
 449                         pr_err("Unable to recompile block at PC 0x%x\n", pc);
 450                         lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
 451                         return NULL;
 452                 }
 453
 454                 lightrec_register_block(state->block_cache, block);
 455         }
 456
 457         return block;
 458 }
 459
 460 static void * get_next_block_func(struct lightrec_state *state, u32 pc)
 461 {
 462         struct block *block;
 463         bool should_recompile;
 464         void *func;
 465
 466         for (;;) {
 467                 func = state->code_lut[lut_offset(pc)];
 468                 if (func && func != state->get_next_block)
 469                         return func;
 470
 471                 block = lightrec_get_block(state, pc);
 472
 473                 if (unlikely(!block))
 474                         return NULL;
 475
 476                 should_recompile = block->flags & BLOCK_SHOULD_RECOMPILE &&
 477                         !(block->flags & BLOCK_IS_DEAD);
 478
 479                 if (unlikely(should_recompile)) {
 480                         pr_debug("Block at PC 0x%08x should recompile\n", pc);
 481
 482                         lightrec_unregister(MEM_FOR_CODE, block->code_size);
 483
 484                         if (ENABLE_THREADED_COMPILER)
 485                                 lightrec_recompiler_add(state->rec, block);
 486                         else
 487                                 lightrec_compile_block(block);
 488                 }
 489
 490                 if (ENABLE_THREADED_COMPILER && likely(!should_recompile))
 491                         func = lightrec_recompiler_run_first_pass(block, &pc);
 492                 else
 493                         func = block->function;
 494
 495                 if (likely(func))
 496                         return func;
 497
 498                 /* Block wasn't compiled yet - run the interpreter */
 499                 if (!ENABLE_THREADED_COMPILER &&
 500                     ((ENABLE_FIRST_PASS && likely(!should_recompile)) ||
 501                      unlikely(block->flags & BLOCK_NEVER_COMPILE)))
 502                         pc = lightrec_emulate_block(block, pc);
 503
 504                 if (likely(!(block->flags & BLOCK_NEVER_COMPILE))) {
 505                         /* Then compile it using the profiled data */
 506                         if (ENABLE_THREADED_COMPILER)
 507                                 lightrec_recompiler_add(state->rec, block);
 508                         else
 509                                 lightrec_compile_block(block);
 510                 }
 511
 512                 if (state->exit_flags != LIGHTREC_EXIT_NORMAL ||
 513                     state->current_cycle >= state->target_cycle) {
 514                         state->next_pc = pc;
 515                         return NULL;
 516                 }
 517         }
 518 }
 519
 520 static s32 c_generic_function_wrapper(struct lightrec_state *state,
 521                                       s32 cycles_delta,
 522                                       void (*f)(struct lightrec_state *,
 523                                                 struct opcode *,
 524                                                 struct block *),
 525                                       struct opcode *op, struct block *block)
 526 {
 527         state->current_cycle = state->target_cycle - cycles_delta;
 528
 529         (*f)(state, op, block);
 530
 531         return state->target_cycle - state->current_cycle;
 532 }
 533
 534 static s32 c_function_wrapper(struct lightrec_state *state, s32 cycles_delta,
 535                               void (*f)(struct lightrec_state *, union code),
 536                               union code op)
 537 {
 538         state->current_cycle = state->target_cycle - cycles_delta;
 539
 540         (*f)(state, op);
 541
 542         return state->target_cycle - state->current_cycle;
 543 }
 544
 545 static struct block * generate_wrapper(struct lightrec_state *state,
 546                                        void *f, bool generic)
 547 {
 548         struct block *block;
 549         jit_state_t *_jit;
 550         unsigned int i;
 551         int stack_ptr;
 552         jit_word_t code_size;
 553         jit_node_t *to_tramp, *to_fn_epilog;
 554
 555         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
 556         if (!block)
 557                 goto err_no_mem;
 558
 559         _jit = jit_new_state();
 560         if (!_jit)
 561                 goto err_free_block;
 562
 563         jit_name("RW wrapper");
 564         jit_note(__FILE__, __LINE__);
 565
 566         /* Wrapper entry point */
 567         jit_prolog();
 568
 569         stack_ptr = jit_allocai(sizeof(uintptr_t) * NUM_TEMPS);
 570
 571         for (i = 0; i < NUM_TEMPS; i++)
 572                 jit_stxi(stack_ptr + i * sizeof(uintptr_t), JIT_FP, JIT_R(i));
 573
 574         /* Jump to the trampoline */
 575         to_tramp = jit_jmpi();
 576
 577         /* The trampoline will jump back here */
 578         to_fn_epilog = jit_label();
 579
 580         for (i = 0; i < NUM_TEMPS; i++)
 581                 jit_ldxi(JIT_R(i), JIT_FP, stack_ptr + i * sizeof(uintptr_t));
 582
 583         jit_ret();
 584         jit_epilog();
 585
 586         /* Trampoline entry point.
 587          * The sole purpose of the trampoline is to cheese Lightning not to
 588          * save/restore the callee-saved register LIGHTREC_REG_CYCLE, since we
 589          * do want to return to the caller with this register modified. */
 590         jit_prolog();
 591         jit_tramp(256);
 592         jit_patch(to_tramp);
 593
 594         jit_prepare();
 595         jit_pushargr(LIGHTREC_REG_STATE);
 596         jit_pushargr(LIGHTREC_REG_CYCLE);
 597         jit_pushargi((uintptr_t)f);
 598         jit_pushargr(JIT_R0);
 599         if (generic) {
 600                 jit_pushargr(JIT_R1);
 601                 jit_finishi(c_generic_function_wrapper);
 602         } else {
 603                 jit_finishi(c_function_wrapper);
 604         }
 605
 606 #if __WORDSIZE == 64
 607         jit_retval_i(LIGHTREC_REG_CYCLE);
 608 #else
 609         jit_retval(LIGHTREC_REG_CYCLE);
 610 #endif
 611
 612         jit_patch_at(jit_jmpi(), to_fn_epilog);
 613         jit_epilog();
 614
 615         block->state = state;
 616         block->_jit = _jit;
 617         block->function = jit_emit();
 618         block->opcode_list = NULL;
 619         block->flags = 0;
 620         block->nb_ops = 0;
 621
 622         jit_get_code(&code_size);
 623         lightrec_register(MEM_FOR_CODE, code_size);
 624
 625         block->code_size = code_size;
 626
 627         if (ENABLE_DISASSEMBLER) {
 628                 pr_debug("Wrapper block:\n");
 629                 jit_disassemble();
 630         }
 631
 632         jit_clear_state();
 633         return block;
 634
 635 err_free_block:
 636         lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
 637 err_no_mem:
 638         pr_err("Unable to compile wrapper: Out of memory\n");
 639         return NULL;
 640 }
 641
 642 static struct block * generate_dispatcher(struct lightrec_state *state)
 643 {
 644         struct block *block;
 645         jit_state_t *_jit;
 646         jit_node_t *to_end, *to_end2, *to_c, *loop, *addr, *addr2;
 647         unsigned int i;
 648         u32 offset, ram_len;
 649         jit_word_t code_size;
 650
 651         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
 652         if (!block)
 653                 goto err_no_mem;
 654
 655         _jit = jit_new_state();
 656         if (!_jit)
 657                 goto err_free_block;
 658
 659         jit_name("dispatcher");
 660         jit_note(__FILE__, __LINE__);
 661
 662         jit_prolog();
 663         jit_frame(256);
 664
 665         jit_getarg(JIT_R0, jit_arg());
 666 #if __WORDSIZE == 64
 667         jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg());
 668 #else
 669         jit_getarg(LIGHTREC_REG_CYCLE, jit_arg());
 670 #endif
 671
 672         /* Force all callee-saved registers to be pushed on the stack */
 673         for (i = 0; i < NUM_REGS; i++)
 674                 jit_movr(JIT_V(i), JIT_V(i));
 675
 676         /* Pass lightrec_state structure to blocks, using the last callee-saved
 677          * register that Lightning provides */
 678         jit_movi(LIGHTREC_REG_STATE, (intptr_t) state);
 679
 680         loop = jit_label();
 681
 682         /* Call the block's code */
 683         jit_jmpr(JIT_R0);
 684
 685         /* The block will jump here, with the number of cycles remaining in
 686          * LIGHTREC_REG_CYCLE */
 687         addr2 = jit_indirect();
 688
 689         /* Jump to end if state->target_cycle < state->current_cycle */
 690         to_end = jit_blei(LIGHTREC_REG_CYCLE, 0);
 691
 692         /* Convert next PC to KUNSEG and avoid mirrors */
 693         ram_len = state->maps[PSX_MAP_KERNEL_USER_RAM].length;
 694         jit_andi(JIT_R0, JIT_V0, 0x10000000 | (ram_len - 1));
 695         to_c = jit_bgei(JIT_R0, ram_len);
 696
 697         /* Fast path: code is running from RAM, use the code LUT */
 698 #if __WORDSIZE == 64
 699         jit_lshi(JIT_R0, JIT_R0, 1);
 700 #endif
 701         jit_addr(JIT_R0, JIT_R0, LIGHTREC_REG_STATE);
 702         jit_ldxi(JIT_R0, JIT_R0, offsetof(struct lightrec_state, code_lut));
 703
 704         /* If we get non-NULL, loop */
 705         jit_patch_at(jit_bnei(JIT_R0, 0), loop);
 706
 707         /* Slow path: call C function get_next_block_func() */
 708         jit_patch(to_c);
 709
 710         if (ENABLE_FIRST_PASS) {
 711                 /* We may call the interpreter - update state->current_cycle */
 712                 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
 713                            offsetof(struct lightrec_state, target_cycle));
 714                 jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE);
 715                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
 716                            LIGHTREC_REG_STATE, JIT_R1);
 717         }
 718
 719         /* The code LUT will be set to this address when the block at the target
 720          * PC has been preprocessed but not yet compiled by the threaded
 721          * recompiler */
 722         addr = jit_indirect();
 723
 724         /* Get the next block */
 725         jit_prepare();
 726         jit_pushargr(LIGHTREC_REG_STATE);
 727         jit_pushargr(JIT_V0);
 728         jit_finishi(&get_next_block_func);
 729         jit_retval(JIT_R0);
 730
 731         if (ENABLE_FIRST_PASS) {
 732                 /* The interpreter may have updated state->current_cycle and
 733                  * state->target_cycle - recalc the delta */
 734                 jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE,
 735                            offsetof(struct lightrec_state, current_cycle));
 736                 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
 737                            offsetof(struct lightrec_state, target_cycle));
 738                 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1);
 739         }
 740
 741         /* If we get non-NULL, loop */
 742         jit_patch_at(jit_bnei(JIT_R0, 0), loop);
 743
 744         to_end2 = jit_jmpi();
 745
 746         /* When exiting, the recompiled code will jump to that address */
 747         jit_note(__FILE__, __LINE__);
 748         jit_patch(to_end);
 749
 750         /* Store back the next_pc to the lightrec_state structure */
 751         offset = offsetof(struct lightrec_state, next_pc);
 752         jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0);
 753
 754         jit_patch(to_end2);
 755
 756         jit_retr(LIGHTREC_REG_CYCLE);
 757         jit_epilog();
 758
 759         block->state = state;
 760         block->_jit = _jit;
 761         block->function = jit_emit();
 762         block->opcode_list = NULL;
 763         block->flags = 0;
 764         block->nb_ops = 0;
 765
 766         jit_get_code(&code_size);
 767         lightrec_register(MEM_FOR_CODE, code_size);
 768
 769         block->code_size = code_size;
 770
 771         state->eob_wrapper_func = jit_address(addr2);
 772         state->get_next_block = jit_address(addr);
 773
 774         if (ENABLE_DISASSEMBLER) {
 775                 pr_debug("Dispatcher block:\n");
 776                 jit_disassemble();
 777         }
 778
 779         /* We're done! */
 780         jit_clear_state();
 781         return block;
 782
 783 err_free_block:
 784         lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
 785 err_no_mem:
 786         pr_err("Unable to compile dispatcher: Out of memory\n");
 787         return NULL;
 788 }
 789
 790 union code lightrec_read_opcode(struct lightrec_state *state, u32 pc)
 791 {
 792         u32 addr, kunseg_pc = kunseg(pc);
 793         const u32 *code;
 794         const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
 795
 796         addr = kunseg_pc - map->pc;
 797
 798         while (map->mirror_of)
 799                 map = map->mirror_of;
 800
 801         code = map->address + addr;
 802
 803         return (union code) *code;
 804 }
 805
 806 static struct block * lightrec_precompile_block(struct lightrec_state *state,
 807                                                 u32 pc)
 808 {
 809         struct opcode *list;
 810         struct block *block;
 811         const u32 *code;
 812         u32 addr, kunseg_pc = kunseg(pc);
 813         const struct lightrec_mem_map *map = lightrec_get_map(state, kunseg_pc);
 814         unsigned int length;
 815
 816         if (!map)
 817                 return NULL;
 818
 819         addr = kunseg_pc - map->pc;
 820
 821         while (map->mirror_of)
 822                 map = map->mirror_of;
 823
 824         code = map->address + addr;
 825
 826         block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
 827         if (!block) {
 828                 pr_err("Unable to recompile block: Out of memory\n");
 829                 return NULL;
 830         }
 831
 832         list = lightrec_disassemble(state, code, &length);
 833         if (!list) {
 834                 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
 835                 return NULL;
 836         }
 837
 838         block->pc = pc;
 839         block->state = state;
 840         block->_jit = NULL;
 841         block->function = NULL;
 842         block->opcode_list = list;
 843         block->map = map;
 844         block->next = NULL;
 845         block->flags = 0;
 846         block->code_size = 0;
 847 #if ENABLE_THREADED_COMPILER
 848         block->op_list_freed = (atomic_flag)ATOMIC_FLAG_INIT;
 849 #endif
 850         block->nb_ops = length / sizeof(u32);
 851
 852         lightrec_optimize(block);
 853
 854         length = block->nb_ops * sizeof(u32);
 855
 856         lightrec_register(MEM_FOR_MIPS_CODE, length);
 857
 858         if (ENABLE_DISASSEMBLER) {
 859                 pr_debug("Disassembled block at PC: 0x%x\n", block->pc);
 860                 lightrec_print_disassembly(block, code, length);
 861         }
 862
 863         pr_debug("Block size: %lu opcodes\n", block->nb_ops);
 864
 865         /* If the first opcode is an 'impossible' branch, never compile the
 866          * block */
 867         if (list->flags & LIGHTREC_EMULATE_BRANCH)
 868                 block->flags |= BLOCK_NEVER_COMPILE;
 869
 870         block->hash = lightrec_calculate_block_hash(block);
 871
 872         pr_debug("Recompile count: %u\n", state->nb_precompile++);
 873
 874         return block;
 875 }
 876
 877 static bool lightrec_block_is_fully_tagged(struct block *block)
 878 {
 879         struct opcode *op;
 880
 881         for (op = block->opcode_list; op; op = op->next) {
 882                 /* Verify that all load/stores of the opcode list
 883                  * Check all loads/stores of the opcode list and mark the
 884                  * block as fully compiled if they all have been tagged. */
 885                 switch (op->c.i.op) {
 886                 case OP_LB:
 887                 case OP_LH:
 888                 case OP_LWL:
 889                 case OP_LW:
 890                 case OP_LBU:
 891                 case OP_LHU:
 892                 case OP_LWR:
 893                 case OP_SB:
 894                 case OP_SH:
 895                 case OP_SWL:
 896                 case OP_SW:
 897                 case OP_SWR:
 898                 case OP_LWC2:
 899                 case OP_SWC2:
 900                         if (!(op->flags & (LIGHTREC_DIRECT_IO |
 901                                            LIGHTREC_HW_IO)))
 902                                 return false;
 903                 default: /* fall-through */
 904                         continue;
 905                 }
 906         }
 907
 908         return true;
 909 }
 910
 911 static void lightrec_reap_block(void *data)
 912 {
 913         struct block *block = data;
 914
 915         pr_debug("Reap dead block at PC 0x%08x\n", block->pc);
 916         lightrec_free_block(block);
 917 }
 918
 919 static void lightrec_reap_jit(void *data)
 920 {
 921         _jit_destroy_state(data);
 922 }
 923
 924 int lightrec_compile_block(struct block *block)
 925 {
 926         struct lightrec_state *state = block->state;
 927         struct lightrec_branch_target *target;
 928         bool op_list_freed = false, fully_tagged = false;
 929         struct block *block2;
 930         struct opcode *elm;
 931         jit_state_t *_jit, *oldjit;
 932         jit_node_t *start_of_block;
 933         bool skip_next = false;
 934         jit_word_t code_size;
 935         unsigned int i, j;
 936         u32 next_pc, offset;
 937
 938         fully_tagged = lightrec_block_is_fully_tagged(block);
 939         if (fully_tagged)
 940                 block->flags |= BLOCK_FULLY_TAGGED;
 941
 942         _jit = jit_new_state();
 943         if (!_jit)
 944                 return -ENOMEM;
 945
 946         oldjit = block->_jit;
 947         block->_jit = _jit;
 948
 949         lightrec_regcache_reset(state->reg_cache);
 950         state->cycles = 0;
 951         state->nb_branches = 0;
 952         state->nb_local_branches = 0;
 953         state->nb_targets = 0;
 954
 955         jit_prolog();
 956         jit_tramp(256);
 957
 958         start_of_block = jit_label();
 959
 960         for (elm = block->opcode_list; elm; elm = elm->next) {
 961                 next_pc = block->pc + elm->offset * sizeof(u32);
 962
 963                 if (skip_next) {
 964                         skip_next = false;
 965                         continue;
 966                 }
 967
 968                 state->cycles += lightrec_cycles_of_opcode(elm->c);
 969
 970                 if (elm->flags & LIGHTREC_EMULATE_BRANCH) {
 971                         pr_debug("Branch at offset 0x%x will be emulated\n",
 972                                  elm->offset << 2);
 973                         lightrec_emit_eob(block, elm, next_pc);
 974                         skip_next = !(elm->flags & LIGHTREC_NO_DS);
 975                 } else if (elm->opcode) {
 976                         lightrec_rec_opcode(block, elm, next_pc);
 977                         skip_next = has_delay_slot(elm->c) &&
 978                                 !(elm->flags & LIGHTREC_NO_DS);
 979 #if _WIN32
 980                         /* FIXME: GNU Lightning on Windows seems to use our
 981                          * mapped registers as temporaries. Until the actual bug
 982                          * is found and fixed, unconditionally mark our
 983                          * registers as live here. */
 984                         lightrec_regcache_mark_live(state->reg_cache, _jit);
 985 #endif
 986                 }
 987         }
 988
 989         for (i = 0; i < state->nb_branches; i++)
 990                 jit_patch(state->branches[i]);
 991
 992         for (i = 0; i < state->nb_local_branches; i++) {
 993                 struct lightrec_branch *branch = &state->local_branches[i];
 994
 995                 pr_debug("Patch local branch to offset 0x%x\n",
 996                          branch->target << 2);
 997
 998                 if (branch->target == 0) {
 999                         jit_patch_at(branch->branch, start_of_block);
1000                         continue;
1001                 }
1002
1003                 for (j = 0; j < state->nb_targets; j++) {
1004                         if (state->targets[j].offset == branch->target) {
1005                                 jit_patch_at(branch->branch,
1006                                              state->targets[j].label);
1007                                 break;
1008                         }
1009                 }
1010
1011                 if (j == state->nb_targets)
1012                         pr_err("Unable to find branch target\n");
1013         }
1014
1015         jit_ldxi(JIT_R0, LIGHTREC_REG_STATE,
1016                  offsetof(struct lightrec_state, eob_wrapper_func));
1017
1018         jit_jmpr(JIT_R0);
1019
1020         jit_ret();
1021         jit_epilog();
1022
1023         block->function = jit_emit();
1024         block->flags &= ~BLOCK_SHOULD_RECOMPILE;
1025
1026         /* Add compiled function to the LUT */
1027         state->code_lut[lut_offset(block->pc)] = block->function;
1028
1029         /* Fill code LUT with the block's entry points */
1030         for (i = 0; i < state->nb_targets; i++) {
1031                 target = &state->targets[i];
1032
1033                 if (target->offset) {
1034                         offset = lut_offset(block->pc) + target->offset;
1035                         state->code_lut[offset] = jit_address(target->label);
1036                 }
1037         }
1038
1039         /* Detect old blocks that have been covered by the new one */
1040         for (i = 0; i < state->nb_targets; i++) {
1041                 target = &state->targets[i];
1042
1043                 if (!target->offset)
1044                         continue;
1045
1046                 offset = block->pc + target->offset * sizeof(u32);
1047                 block2 = lightrec_find_block(state->block_cache, offset);
1048                 if (block2) {
1049                         /* No need to check if block2 is compilable - it must
1050                          * be, otherwise block wouldn't be compilable either */
1051
1052                         block2->flags |= BLOCK_IS_DEAD;
1053
1054                         pr_debug("Reap block 0x%08x as it's covered by block "
1055                                  "0x%08x\n", block2->pc, block->pc);
1056
1057                         lightrec_unregister_block(state->block_cache, block2);
1058
1059                         if (ENABLE_THREADED_COMPILER) {
1060                                 lightrec_recompiler_remove(state->rec, block2);
1061                                 lightrec_reaper_add(state->reaper,
1062                                                     lightrec_reap_block,
1063                                                     block2);
1064                         } else {
1065                                 lightrec_free_block(block2);
1066                         }
1067                 }
1068         }
1069
1070         jit_get_code(&code_size);
1071         lightrec_register(MEM_FOR_CODE, code_size);
1072
1073         block->code_size = code_size;
1074
1075         if (ENABLE_DISASSEMBLER) {
1076                 pr_debug("Compiling block at PC: 0x%x\n", block->pc);
1077                 jit_disassemble();
1078         }
1079
1080         jit_clear_state();
1081
1082 #if ENABLE_THREADED_COMPILER
1083         if (fully_tagged)
1084                 op_list_freed = atomic_flag_test_and_set(&block->op_list_freed);
1085 #endif
1086         if (fully_tagged && !op_list_freed) {
1087                 pr_debug("Block PC 0x%08x is fully tagged"
1088                          " - free opcode list\n", block->pc);
1089                 lightrec_free_opcode_list(state, block->opcode_list);
1090                 block->opcode_list = NULL;
1091         }
1092
1093         if (oldjit) {
1094                 pr_debug("Block 0x%08x recompiled, reaping old jit context.\n",
1095                          block->pc);
1096
1097                 if (ENABLE_THREADED_COMPILER)
1098                         lightrec_reaper_add(state->reaper,
1099                                             lightrec_reap_jit, oldjit);
1100                 else
1101                         _jit_destroy_state(oldjit);
1102         }
1103
1104         return 0;
1105 }
1106
1107 u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle)
1108 {
1109         s32 (*func)(void *, s32) = (void *)state->dispatcher->function;
1110         void *block_trace;
1111         s32 cycles_delta;
1112
1113         state->exit_flags = LIGHTREC_EXIT_NORMAL;
1114
1115         /* Handle the cycle counter overflowing */
1116         if (unlikely(target_cycle < state->current_cycle))
1117                 target_cycle = UINT_MAX;
1118
1119         state->target_cycle = target_cycle;
1120
1121         block_trace = get_next_block_func(state, pc);
1122         if (block_trace) {
1123                 cycles_delta = state->target_cycle - state->current_cycle;
1124
1125                 cycles_delta = (*func)(block_trace, cycles_delta);
1126
1127                 state->current_cycle = state->target_cycle - cycles_delta;
1128         }
1129
1130         if (ENABLE_THREADED_COMPILER)
1131                 lightrec_reaper_reap(state->reaper);
1132
1133         return state->next_pc;
1134 }
1135
1136 u32 lightrec_execute_one(struct lightrec_state *state, u32 pc)
1137 {
1138         return lightrec_execute(state, pc, state->current_cycle);
1139 }
1140
1141 u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc)
1142 {
1143         struct block *block = lightrec_get_block(state, pc);
1144         if (!block)
1145                 return 0;
1146
1147         state->exit_flags = LIGHTREC_EXIT_NORMAL;
1148
1149         return lightrec_emulate_block(block, pc);
1150 }
1151
1152 void lightrec_free_block(struct block *block)
1153 {
1154         lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32));
1155         if (block->opcode_list)
1156                 lightrec_free_opcode_list(block->state, block->opcode_list);
1157         if (block->_jit)
1158                 _jit_destroy_state(block->_jit);
1159         lightrec_unregister(MEM_FOR_CODE, block->code_size);
1160         lightrec_free(block->state, MEM_FOR_IR, sizeof(*block), block);
1161 }
1162
1163 struct lightrec_state * lightrec_init(char *argv0,
1164                                       const struct lightrec_mem_map *map,
1165                                       size_t nb,
1166                                       const struct lightrec_ops *ops)
1167 {
1168         struct lightrec_state *state;
1169
1170         /* Sanity-check ops */
1171         if (!ops ||
1172             !ops->cop0_ops.mfc || !ops->cop0_ops.cfc || !ops->cop0_ops.mtc ||
1173             !ops->cop0_ops.ctc || !ops->cop0_ops.op ||
1174             !ops->cop2_ops.mfc || !ops->cop2_ops.cfc || !ops->cop2_ops.mtc ||
1175             !ops->cop2_ops.ctc || !ops->cop2_ops.op) {
1176                 pr_err("Missing callbacks in lightrec_ops structure\n");
1177                 return NULL;
1178         }
1179
1180         init_jit(argv0);
1181
1182         state = calloc(1, sizeof(*state) +
1183                        sizeof(*state->code_lut) * CODE_LUT_SIZE);
1184         if (!state)
1185                 goto err_finish_jit;
1186
1187         lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) +
1188                           sizeof(*state->code_lut) * CODE_LUT_SIZE);
1189
1190 #if ENABLE_TINYMM
1191         state->tinymm = tinymm_init(malloc, free, 4096);
1192         if (!state->tinymm)
1193                 goto err_free_state;
1194 #endif
1195
1196         state->block_cache = lightrec_blockcache_init(state);
1197         if (!state->block_cache)
1198                 goto err_free_tinymm;
1199
1200         state->reg_cache = lightrec_regcache_init(state);
1201         if (!state->reg_cache)
1202                 goto err_free_block_cache;
1203
1204         if (ENABLE_THREADED_COMPILER) {
1205                 state->rec = lightrec_recompiler_init(state);
1206                 if (!state->rec)
1207                         goto err_free_reg_cache;
1208
1209                 state->reaper = lightrec_reaper_init(state);
1210                 if (!state->reaper)
1211                         goto err_free_recompiler;
1212         }
1213
1214         state->nb_maps = nb;
1215         state->maps = map;
1216
1217         memcpy(&state->ops, ops, sizeof(*ops));
1218
1219         state->dispatcher = generate_dispatcher(state);
1220         if (!state->dispatcher)
1221                 goto err_free_reaper;
1222
1223         state->rw_generic_wrapper = generate_wrapper(state,
1224                                                      lightrec_rw_generic_cb,
1225                                                      true);
1226         if (!state->rw_generic_wrapper)
1227                 goto err_free_dispatcher;
1228
1229         state->rw_wrapper = generate_wrapper(state, lightrec_rw_cb, false);
1230         if (!state->rw_wrapper)
1231                 goto err_free_generic_rw_wrapper;
1232
1233         state->mfc_wrapper = generate_wrapper(state, lightrec_mfc_cb, false);
1234         if (!state->mfc_wrapper)
1235                 goto err_free_rw_wrapper;
1236
1237         state->mtc_wrapper = generate_wrapper(state, lightrec_mtc_cb, false);
1238         if (!state->mtc_wrapper)
1239                 goto err_free_mfc_wrapper;
1240
1241         state->rfe_wrapper = generate_wrapper(state, lightrec_rfe_cb, false);
1242         if (!state->rfe_wrapper)
1243                 goto err_free_mtc_wrapper;
1244
1245         state->cp_wrapper = generate_wrapper(state, lightrec_cp_cb, false);
1246         if (!state->cp_wrapper)
1247                 goto err_free_rfe_wrapper;
1248
1249         state->syscall_wrapper = generate_wrapper(state, lightrec_syscall_cb,
1250                                                   false);
1251         if (!state->syscall_wrapper)
1252                 goto err_free_cp_wrapper;
1253
1254         state->break_wrapper = generate_wrapper(state, lightrec_break_cb,
1255                                                 false);
1256         if (!state->break_wrapper)
1257                 goto err_free_syscall_wrapper;
1258
1259         state->rw_generic_func = state->rw_generic_wrapper->function;
1260         state->rw_func = state->rw_wrapper->function;
1261         state->mfc_func = state->mfc_wrapper->function;
1262         state->mtc_func = state->mtc_wrapper->function;
1263         state->rfe_func = state->rfe_wrapper->function;
1264         state->cp_func = state->cp_wrapper->function;
1265         state->syscall_func = state->syscall_wrapper->function;
1266         state->break_func = state->break_wrapper->function;
1267
1268         map = &state->maps[PSX_MAP_BIOS];
1269         state->offset_bios = (uintptr_t)map->address - map->pc;
1270
1271         map = &state->maps[PSX_MAP_SCRATCH_PAD];
1272         state->offset_scratch = (uintptr_t)map->address - map->pc;
1273
1274         map = &state->maps[PSX_MAP_KERNEL_USER_RAM];
1275         state->offset_ram = (uintptr_t)map->address - map->pc;
1276
1277         if (state->maps[PSX_MAP_MIRROR1].address == map->address + 0x200000 &&
1278             state->maps[PSX_MAP_MIRROR2].address == map->address + 0x400000 &&
1279             state->maps[PSX_MAP_MIRROR3].address == map->address + 0x600000)
1280                 state->mirrors_mapped = true;
1281
1282         return state;
1283
1284 err_free_syscall_wrapper:
1285         lightrec_free_block(state->syscall_wrapper);
1286 err_free_cp_wrapper:
1287         lightrec_free_block(state->cp_wrapper);
1288 err_free_rfe_wrapper:
1289         lightrec_free_block(state->rfe_wrapper);
1290 err_free_mtc_wrapper:
1291         lightrec_free_block(state->mtc_wrapper);
1292 err_free_mfc_wrapper:
1293         lightrec_free_block(state->mfc_wrapper);
1294 err_free_rw_wrapper:
1295         lightrec_free_block(state->rw_wrapper);
1296 err_free_generic_rw_wrapper:
1297         lightrec_free_block(state->rw_generic_wrapper);
1298 err_free_dispatcher:
1299         lightrec_free_block(state->dispatcher);
1300 err_free_reaper:
1301         if (ENABLE_THREADED_COMPILER)
1302                 lightrec_reaper_destroy(state->reaper);
1303 err_free_recompiler:
1304         if (ENABLE_THREADED_COMPILER)
1305                 lightrec_free_recompiler(state->rec);
1306 err_free_reg_cache:
1307         lightrec_free_regcache(state->reg_cache);
1308 err_free_block_cache:
1309         lightrec_free_block_cache(state->block_cache);
1310 err_free_tinymm:
1311 #if ENABLE_TINYMM
1312         tinymm_shutdown(state->tinymm);
1313 err_free_state:
1314 #endif
1315         lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1316                             sizeof(*state->code_lut) * CODE_LUT_SIZE);
1317         free(state);
1318 err_finish_jit:
1319         finish_jit();
1320         return NULL;
1321 }
1322
1323 void lightrec_destroy(struct lightrec_state *state)
1324 {
1325         if (ENABLE_THREADED_COMPILER) {
1326                 lightrec_free_recompiler(state->rec);
1327                 lightrec_reaper_destroy(state->reaper);
1328         }
1329
1330         lightrec_free_regcache(state->reg_cache);
1331         lightrec_free_block_cache(state->block_cache);
1332         lightrec_free_block(state->dispatcher);
1333         lightrec_free_block(state->rw_generic_wrapper);
1334         lightrec_free_block(state->rw_wrapper);
1335         lightrec_free_block(state->mfc_wrapper);
1336         lightrec_free_block(state->mtc_wrapper);
1337         lightrec_free_block(state->rfe_wrapper);
1338         lightrec_free_block(state->cp_wrapper);
1339         lightrec_free_block(state->syscall_wrapper);
1340         lightrec_free_block(state->break_wrapper);
1341         finish_jit();
1342
1343 #if ENABLE_TINYMM
1344         tinymm_shutdown(state->tinymm);
1345 #endif
1346         lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
1347                             sizeof(*state->code_lut) * CODE_LUT_SIZE);
1348         free(state);
1349 }
1350
1351 void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len)
1352 {
1353         u32 kaddr = kunseg(addr & ~0x3);
1354         const struct lightrec_mem_map *map = lightrec_get_map(state, kaddr);
1355
1356         if (map) {
1357                 while (map->mirror_of)
1358                         map = map->mirror_of;
1359
1360                 if (map != &state->maps[PSX_MAP_KERNEL_USER_RAM])
1361                         return;
1362
1363                 /* Handle mirrors */
1364                 kaddr &= (state->maps[PSX_MAP_KERNEL_USER_RAM].length - 1);
1365
1366                 for (; len > 4; len -= 4, kaddr += 4)
1367                         lightrec_invalidate_map(state, map, kaddr);
1368
1369                 lightrec_invalidate_map(state, map, kaddr);
1370         }
1371 }
1372
1373 void lightrec_invalidate_all(struct lightrec_state *state)
1374 {
1375         memset(state->code_lut, 0, sizeof(*state->code_lut) * CODE_LUT_SIZE);
1376 }
1377
1378 void lightrec_set_invalidate_mode(struct lightrec_state *state, bool dma_only)
1379 {
1380         if (state->invalidate_from_dma_only != dma_only)
1381                 lightrec_invalidate_all(state);
1382
1383         state->invalidate_from_dma_only = dma_only;
1384 }
1385
1386 void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags)
1387 {
1388         if (flags != LIGHTREC_EXIT_NORMAL) {
1389                 state->exit_flags |= flags;
1390                 state->target_cycle = state->current_cycle;
1391         }
1392 }
1393
1394 u32 lightrec_exit_flags(struct lightrec_state *state)
1395 {
1396         return state->exit_flags;
1397 }
1398
1399 void lightrec_dump_registers(struct lightrec_state *state, u32 regs[34])
1400 {
1401         memcpy(regs, state->native_reg_cache, sizeof(state->native_reg_cache));
1402 }
1403
1404 void lightrec_restore_registers(struct lightrec_state *state, u32 regs[34])
1405 {
1406         memcpy(state->native_reg_cache, regs, sizeof(state->native_reg_cache));
1407 }
1408
1409 u32 lightrec_current_cycle_count(const struct lightrec_state *state)
1410 {
1411         return state->current_cycle;
1412 }
1413
1414 void lightrec_reset_cycle_count(struct lightrec_state *state, u32 cycles)
1415 {
1416         state->current_cycle = cycles;
1417
1418         if (state->target_cycle < cycles)
1419                 state->target_cycle = cycles;
1420 }
1421
1422 void lightrec_set_target_cycle_count(struct lightrec_state *state, u32 cycles)
1423 {
1424         if (state->exit_flags == LIGHTREC_EXIT_NORMAL) {
1425                 if (cycles < state->current_cycle)
1426                         cycles = state->current_cycle;
1427
1428                 state->target_cycle = cycles;
1429         }
1430 }