notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update analog default list
[pcsx_rearmed.git] / deps / lightrec / lightrec.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC		 4 */
5
6#include "blockcache.h"
d16005f8
PC		 7#include "debug.h"
8#include "disassembler.h"
9#include "emitter.h"
10#include "interpreter.h"
98fa08a5
PC		 11#include "lightrec-config.h"
12#include "lightning-wrapper.h"
d16005f8
PC		 13#include "lightrec.h"
14#include "memmanager.h"
a59e5536 15#include "reaper.h"
d16005f8
PC		 16#include "recompiler.h"
17#include "regcache.h"
18#include "optimizer.h"
02487de7 19#include "tlsf/tlsf.h"
d16005f8
PC		 20
21#include <errno.h>
98fa08a5 22#include <inttypes.h>
d16005f8
PC		 23#include <limits.h>
24#if ENABLE_THREADED_COMPILER
25#include <stdatomic.h>
26#endif
27#include <stdbool.h>
28#include <stddef.h>
29#include <string.h>
d16005f8 30
d16005f8
PC		 31static struct block * lightrec_precompile_block(struct lightrec_state *state,
32 u32 pc);
98fa08a5
PC		 33static bool lightrec_block_is_fully_tagged(const struct block *block);
34
35static void lightrec_mtc2(struct lightrec_state *state, u8 reg, u32 data);
36static u32 lightrec_mfc2(struct lightrec_state *state, u8 reg);
d16005f8 37
a59e5536 38static void lightrec_default_sb(struct lightrec_state *state, u32 opcode,
2e6c828e 39 void *host, u32 addr, u32 data)
a59e5536 40{
2e6c828e 41 *(u8 *)host = (u8)data;
a59e5536 42
684432ad 43 if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY))
a59e5536 44 lightrec_invalidate(state, addr, 1);
45}
46
47static void lightrec_default_sh(struct lightrec_state *state, u32 opcode,
2e6c828e 48 void *host, u32 addr, u32 data)
a59e5536 49{
2e6c828e 50 *(u16 *)host = HTOLE16((u16)data);
a59e5536 51
684432ad 52 if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY))
a59e5536 53 lightrec_invalidate(state, addr, 2);
54}
55
56static void lightrec_default_sw(struct lightrec_state *state, u32 opcode,
57 void *host, u32 addr, u32 data)
58{
59 *(u32 *)host = HTOLE32(data);
60
684432ad 61 if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY))
a59e5536 62 lightrec_invalidate(state, addr, 4);
63}
64
65static u8 lightrec_default_lb(struct lightrec_state *state,
66 u32 opcode, void *host, u32 addr)
67{
68 return *(u8 *)host;
69}
70
71static u16 lightrec_default_lh(struct lightrec_state *state,
72 u32 opcode, void *host, u32 addr)
73{
74 return LE16TOH(*(u16 *)host);
75}
76
77static u32 lightrec_default_lw(struct lightrec_state *state,
78 u32 opcode, void *host, u32 addr)
79{
80 return LE32TOH(*(u32 *)host);
81}
82
5459088b
PC		 83static u32 lightrec_default_lwu(struct lightrec_state *state,
84 u32 opcode, void *host, u32 addr)
85{
86 u32 val;
87
88 memcpy(&val, host, 4);
89
90 return LE32TOH(val);
91}
92
93static void lightrec_default_swu(struct lightrec_state *state, u32 opcode,
94 void *host, u32 addr, u32 data)
95{
96 data = HTOLE32(data);
97
98 memcpy(host, &data, 4);
99
100 if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY))
101 lightrec_invalidate(state, addr & ~0x3, 8);
102}
103
a59e5536 104static const struct lightrec_mem_map_ops lightrec_default_ops = {
105 .sb = lightrec_default_sb,
106 .sh = lightrec_default_sh,
107 .sw = lightrec_default_sw,
108 .lb = lightrec_default_lb,
109 .lh = lightrec_default_lh,
110 .lw = lightrec_default_lw,
5459088b
PC		 111 .lwu = lightrec_default_lwu,
112 .swu = lightrec_default_swu,
a59e5536 113};
114
98fa08a5
PC		 115static void __segfault_cb(struct lightrec_state *state, u32 addr,
116 const struct block *block)
d16005f8
PC		 117{
118 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
119 pr_err("Segmentation fault in recompiled code: invalid "
f5ee77ca 120 "load/store at address "PC_FMT"\n", addr);
98fa08a5 121 if (block)
f5ee77ca 122 pr_err("Was executing block "PC_FMT"\n", block->pc);
d16005f8
PC		 123}
124
a59e5536 125static void lightrec_swl(struct lightrec_state *state,
126 const struct lightrec_mem_map_ops *ops,
127 u32 opcode, void *host, u32 addr, u32 data)
d16005f8 128{
a59e5536 129 unsigned int shift = addr & 0x3;
13b02197 130 unsigned int mask = shift < 3 ? GENMASK(31, (shift + 1) * 8) : 0;
a59e5536 131 u32 old_data;
132
133 /* Align to 32 bits */
134 addr &= ~3;
135 host = (void *)((uintptr_t)host & ~3);
136
137 old_data = ops->lw(state, opcode, host, addr);
138
139 data = (data >> ((3 - shift) * 8)) | (old_data & mask);
140
141 ops->sw(state, opcode, host, addr, data);
142}
143
144static void lightrec_swr(struct lightrec_state *state,
145 const struct lightrec_mem_map_ops *ops,
146 u32 opcode, void *host, u32 addr, u32 data)
147{
148 unsigned int shift = addr & 0x3;
149 unsigned int mask = (1 << (shift * 8)) - 1;
150 u32 old_data;
151
152 /* Align to 32 bits */
153 addr &= ~3;
154 host = (void *)((uintptr_t)host & ~3);
155
156 old_data = ops->lw(state, opcode, host, addr);
157
158 data = (data << (shift * 8)) | (old_data & mask);
159
160 ops->sw(state, opcode, host, addr, data);
161}
162
163static void lightrec_swc2(struct lightrec_state *state, union code op,
164 const struct lightrec_mem_map_ops *ops,
165 void *host, u32 addr)
166{
98fa08a5 167 u32 data = lightrec_mfc2(state, op.i.rt);
a59e5536 168
169 ops->sw(state, op.opcode, host, addr, data);
170}
171
172static u32 lightrec_lwl(struct lightrec_state *state,
173 const struct lightrec_mem_map_ops *ops,
174 u32 opcode, void *host, u32 addr, u32 data)
175{
176 unsigned int shift = addr & 0x3;
177 unsigned int mask = (1 << (24 - shift * 8)) - 1;
178 u32 old_data;
179
180 /* Align to 32 bits */
181 addr &= ~3;
182 host = (void *)((uintptr_t)host & ~3);
183
184 old_data = ops->lw(state, opcode, host, addr);
185
186 return (data & mask) | (old_data << (24 - shift * 8));
187}
188
189static u32 lightrec_lwr(struct lightrec_state *state,
190 const struct lightrec_mem_map_ops *ops,
191 u32 opcode, void *host, u32 addr, u32 data)
192{
193 unsigned int shift = addr & 0x3;
13b02197 194 unsigned int mask = shift ? GENMASK(31, 32 - shift * 8) : 0;
a59e5536 195 u32 old_data;
196
197 /* Align to 32 bits */
198 addr &= ~3;
199 host = (void *)((uintptr_t)host & ~3);
200
201 old_data = ops->lw(state, opcode, host, addr);
202
203 return (data & mask) | (old_data >> (shift * 8));
204}
205
206static void lightrec_lwc2(struct lightrec_state *state, union code op,
207 const struct lightrec_mem_map_ops *ops,
208 void *host, u32 addr)
209{
210 u32 data = ops->lw(state, op.opcode, host, addr);
211
98fa08a5 212 lightrec_mtc2(state, op.i.rt, data);
d16005f8
PC		 213}
214
215static void lightrec_invalidate_map(struct lightrec_state *state,
98fa08a5 216 const struct lightrec_mem_map *map, u32 addr, u32 len)
d16005f8 217{
98fa08a5 218 if (map == &state->maps[PSX_MAP_KERNEL_USER_RAM]) {
02487de7
PC		 219 memset(lut_address(state, lut_offset(addr)), 0,
220 ((len + 3) / 4) * lut_elm_size(state));
98fa08a5 221 }
d16005f8
PC		 222}
223
9259d748 224static enum psx_map
02487de7 225lightrec_get_map_idx(struct lightrec_state *state, u32 kaddr)
d16005f8 226{
98fa08a5 227 const struct lightrec_mem_map *map;
d16005f8
PC		 228 unsigned int i;
229
230 for (i = 0; i < state->nb_maps; i++) {
02487de7 231 map = &state->maps[i];
d16005f8 232
02487de7
PC		 233 if (kaddr >= map->pc && kaddr < map->pc + map->length)
234 return (enum psx_map) i;
d16005f8
PC		 235 }
236
02487de7
PC		 237 return PSX_MAP_UNKNOWN;
238}
239
240const struct lightrec_mem_map *
241lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr)
242{
243 const struct lightrec_mem_map *map;
244 enum psx_map idx;
245 u32 addr;
246
247 idx = lightrec_get_map_idx(state, kaddr);
248 if (idx == PSX_MAP_UNKNOWN)
98fa08a5
PC		 249 return NULL;
250
02487de7 251 map = &state->maps[idx];
98fa08a5
PC		 252 addr = kaddr - map->pc;
253
254 while (map->mirror_of)
255 map = map->mirror_of;
256
257 if (host)
258 *host = map->address + addr;
259
260 return map;
d16005f8
PC		 261}
262
cb72ea13
PC		 263u32 lightrec_rw(struct lightrec_state *state, union code op, u32 base,
264 u32 data, u32 *flags, struct block *block, u16 offset)
d16005f8
PC		 265{
266 const struct lightrec_mem_map *map;
a59e5536 267 const struct lightrec_mem_map_ops *ops;
98fa08a5 268 u32 opcode = op.opcode;
cb72ea13
PC		 269 bool was_tagged = true;
270 u16 old_flags;
271 u32 addr;
a59e5536 272 void *host;
d16005f8 273
cb72ea13 274 addr = kunseg(base + (s16) op.i.imm);
d16005f8 275
cb72ea13 276 map = lightrec_get_map(state, &host, addr);
d16005f8 277 if (!map) {
98fa08a5 278 __segfault_cb(state, addr, block);
d16005f8
PC		 279 return 0;
280 }
281
cb72ea13
PC		 282 if (flags)
283 was_tagged = LIGHTREC_FLAGS_GET_IO_MODE(*flags);
d16005f8 284
6ce0b00a 285 if (likely(!map->ops)) {
cb72ea13
PC		 286 if (flags && !LIGHTREC_FLAGS_GET_IO_MODE(*flags)) {
287 /* Force parallel port accesses as HW accesses, because
288 * the direct-I/O emitters can't differenciate it. */
289 if (unlikely(map == &state->maps[PSX_MAP_PARALLEL_PORT]))
290 *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW);
291 /* If the base register is 0x0, be extra suspicious.
292 * Some games (e.g. Sled Storm) actually do segmentation
293 * faults by using uninitialized pointers, which are
294 * later initialized to point to hardware registers. */
295 else if (op.i.rs && base == 0x0)
296 *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW);
297 else
298 *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_DIRECT);
299 }
d16005f8 300
a59e5536 301 ops = &lightrec_default_ops;
6ce0b00a
PC		 302 } else if (flags &&
303 LIGHTREC_FLAGS_GET_IO_MODE(*flags) == LIGHTREC_IO_DIRECT_HW) {
304 ops = &lightrec_default_ops;
305 } else {
306 if (flags && !LIGHTREC_FLAGS_GET_IO_MODE(*flags))
307 *flags |= LIGHTREC_IO_MODE(LIGHTREC_IO_HW);
308
309 ops = map->ops;
a59e5536 310 }
d16005f8 311
cb72ea13
PC		 312 if (!was_tagged) {
313 old_flags = block_set_flags(block, BLOCK_SHOULD_RECOMPILE);
314
315 if (!(old_flags & BLOCK_SHOULD_RECOMPILE)) {
f5ee77ca 316 pr_debug("Opcode of block at "PC_FMT" has been tagged"
cb72ea13
PC		 317 " - flag for recompilation\n", block->pc);
318
319 lut_write(state, lut_offset(block->pc), NULL);
320 }
321 }
322
d16005f8
PC		 323 switch (op.i.op) {
324 case OP_SB:
2e6c828e 325 ops->sb(state, opcode, host, addr, data);
d16005f8
PC		 326 return 0;
327 case OP_SH:
2e6c828e 328 ops->sh(state, opcode, host, addr, data);
d16005f8
PC		 329 return 0;
330 case OP_SWL:
a59e5536 331 lightrec_swl(state, ops, opcode, host, addr, data);
d16005f8
PC		 332 return 0;
333 case OP_SWR:
a59e5536 334 lightrec_swr(state, ops, opcode, host, addr, data);
d16005f8
PC		 335 return 0;
336 case OP_SW:
a59e5536 337 ops->sw(state, opcode, host, addr, data);
d16005f8
PC		 338 return 0;
339 case OP_SWC2:
a59e5536 340 lightrec_swc2(state, op, ops, host, addr);
d16005f8
PC		 341 return 0;
342 case OP_LB:
a59e5536 343 return (s32) (s8) ops->lb(state, opcode, host, addr);
d16005f8 344 case OP_LBU:
a59e5536 345 return ops->lb(state, opcode, host, addr);
d16005f8 346 case OP_LH:
a59e5536 347 return (s32) (s16) ops->lh(state, opcode, host, addr);
d16005f8 348 case OP_LHU:
a59e5536 349 return ops->lh(state, opcode, host, addr);
d16005f8 350 case OP_LWC2:
a59e5536 351 lightrec_lwc2(state, op, ops, host, addr);
d16005f8 352 return 0;
a59e5536 353 case OP_LWL:
354 return lightrec_lwl(state, ops, opcode, host, addr, data);
355 case OP_LWR:
356 return lightrec_lwr(state, ops, opcode, host, addr, data);
5459088b
PC		 357 case OP_META_LWU:
358 return ops->lwu(state, opcode, host, addr);
359 case OP_META_SWU:
360 ops->swu(state, opcode, host, addr, data);
361 return 0;
d16005f8
PC		 362 case OP_LW:
363 default:
a59e5536 364 return ops->lw(state, opcode, host, addr);
d16005f8
PC		 365 }
366}
367
368static void lightrec_rw_helper(struct lightrec_state *state,
03535202 369 union code op, u32 *flags,
cb72ea13 370 struct block *block, u16 offset)
d16005f8 371{
98fa08a5 372 u32 ret = lightrec_rw(state, op, state->regs.gpr[op.i.rs],
cb72ea13 373 state->regs.gpr[op.i.rt], flags, block, offset);
d16005f8
PC		 374
375 switch (op.i.op) {
376 case OP_LB:
377 case OP_LBU:
378 case OP_LH:
379 case OP_LHU:
380 case OP_LWL:
381 case OP_LWR:
382 case OP_LW:
5459088b 383 case OP_META_LWU:
cb72ea13
PC		 384 if (OPT_HANDLE_LOAD_DELAYS && unlikely(!state->in_delay_slot_n)) {
385 state->temp_reg = ret;
386 state->in_delay_slot_n = 0xff;
387 } else if (op.i.rt) {
98fa08a5 388 state->regs.gpr[op.i.rt] = ret;
cb72ea13 389 }
d8b04acd
PC		 390 fallthrough;
391 default:
d16005f8
PC		 392 break;
393 }
394}
395
03535202 396static void lightrec_rw_cb(struct lightrec_state *state, u32 arg)
d16005f8 397{
cb72ea13 398 lightrec_rw_helper(state, (union code) arg, NULL, NULL, 0);
d16005f8
PC		 399}
400
03535202 401static void lightrec_rw_generic_cb(struct lightrec_state *state, u32 arg)
d16005f8 402{
98fa08a5
PC		 403 struct block *block;
404 struct opcode *op;
98fa08a5
PC		 405 u16 offset = (u16)arg;
406
407 block = lightrec_find_block_from_lut(state->block_cache,
0e720fb1 408 arg >> 16, state->curr_pc);
98fa08a5 409 if (unlikely(!block)) {
f5ee77ca 410 pr_err("rw_generic: No block found in LUT for "PC_FMT" offset 0x%"PRIx16"\n",
0e720fb1 411 state->curr_pc, offset);
ba3814c1 412 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
98fa08a5
PC		 413 return;
414 }
415
416 op = &block->opcode_list[offset];
cb72ea13 417 lightrec_rw_helper(state, op->c, &op->flags, block, offset);
d16005f8
PC		 418}
419
98fa08a5 420static u32 clamp_s32(s32 val, s32 min, s32 max)
d16005f8 421{
98fa08a5
PC		 422 return val < min ? min : val > max ? max : val;
423}
d16005f8 424
11357fef
PC		 425static u16 load_u16(u32 *ptr)
426{
427 return ((struct u16x2 *) ptr)->l;
428}
429
430static void store_u16(u32 *ptr, u16 value)
431{
432 ((struct u16x2 *) ptr)->l = value;
433}
434
98fa08a5
PC		 435static u32 lightrec_mfc2(struct lightrec_state *state, u8 reg)
436{
437 s16 gteir1, gteir2, gteir3;
438
439 switch (reg) {
440 case 1:
441 case 3:
442 case 5:
443 case 8:
444 case 9:
445 case 10:
446 case 11:
11357fef 447 return (s32)(s16) load_u16(&state->regs.cp2d[reg]);
98fa08a5
PC		 448 case 7:
449 case 16:
450 case 17:
451 case 18:
452 case 19:
11357fef 453 return load_u16(&state->regs.cp2d[reg]);
98fa08a5
PC		 454 case 28:
455 case 29:
11357fef
PC		 456 gteir1 = (s16) load_u16(&state->regs.cp2d[9]);
457 gteir2 = (s16) load_u16(&state->regs.cp2d[10]);
458 gteir3 = (s16) load_u16(&state->regs.cp2d[11]);
98fa08a5
PC		 459
460 return clamp_s32(gteir1 >> 7, 0, 0x1f) << 0 |
461 clamp_s32(gteir2 >> 7, 0, 0x1f) << 5 |
462 clamp_s32(gteir3 >> 7, 0, 0x1f) << 10;
463 case 15:
464 reg = 14;
d8b04acd
PC		 465 fallthrough;
466 default:
98fa08a5
PC		 467 return state->regs.cp2d[reg];
468 }
469}
d16005f8 470
98fa08a5
PC		 471u32 lightrec_mfc(struct lightrec_state *state, union code op)
472{
ba3814c1
PC		 473 u32 val;
474
98fa08a5
PC		 475 if (op.i.op == OP_CP0)
476 return state->regs.cp0[op.r.rd];
9259d748
PC		 477
478 if (op.i.op == OP_SWC2) {
479 val = lightrec_mfc2(state, op.i.rt);
480 } else if (op.r.rs == OP_CP2_BASIC_MFC2)
fdf33147
PC		 481 val = lightrec_mfc2(state, op.r.rd);
482 else {
483 val = state->regs.cp2c[op.r.rd];
484
485 switch (op.r.rd) {
486 case 4:
487 case 12:
488 case 20:
489 case 26:
490 case 27:
491 case 29:
492 case 30:
493 val = (u32)(s16)val;
494 fallthrough;
495 default:
496 break;
497 }
498 }
ba3814c1 499
fdf33147
PC		 500 if (state->ops.cop2_notify)
501 (*state->ops.cop2_notify)(state, op.opcode, val);
ba3814c1 502
fdf33147
PC		 503 return val;
504}
505
506static void lightrec_mfc_cb(struct lightrec_state *state, union code op)
507{
508 u32 rt = lightrec_mfc(state, op);
509
9259d748 510 if (op.i.op == OP_SWC2)
cb72ea13 511 state->temp_reg = rt;
9259d748 512 else if (op.r.rt)
fdf33147 513 state->regs.gpr[op.r.rt] = rt;
d16005f8
PC		 514}
515
98fa08a5 516static void lightrec_mtc0(struct lightrec_state *state, u8 reg, u32 data)
d16005f8 517{
fd58fa32 518 u32 status, oldstatus, cause;
98fa08a5
PC		 519
520 switch (reg) {
521 case 1:
522 case 4:
523 case 8:
524 case 14:
525 case 15:
526 /* Those registers are read-only */
527 return;
fd58fa32 528 default:
98fa08a5
PC		 529 break;
530 }
d16005f8 531
98fa08a5
PC		 532 if (reg == 12) {
533 status = state->regs.cp0[12];
fd58fa32 534 oldstatus = status;
d16005f8 535
98fa08a5
PC		 536 if (status & ~data & BIT(16)) {
537 state->ops.enable_ram(state, true);
538 lightrec_invalidate_all(state);
539 } else if (~status & data & BIT(16)) {
540 state->ops.enable_ram(state, false);
541 }
542 }
543
fd58fa32
PC		 544 if (reg == 13) {
545 state->regs.cp0[13] &= ~0x300;
546 state->regs.cp0[13] |= data & 0x300;
547 } else {
548 state->regs.cp0[reg] = data;
549 }
98fa08a5
PC		 550
551 if (reg == 12 || reg == 13) {
552 cause = state->regs.cp0[13];
553 status = state->regs.cp0[12];
554
fd58fa32 555 /* Handle software interrupts */
684432ad 556 if ((!!(status & cause & 0x300)) & status)
98fa08a5 557 lightrec_set_exit_flags(state, LIGHTREC_EXIT_CHECK_INTERRUPT);
fd58fa32
PC		 558
559 /* Handle hardware interrupts */
560 if (reg == 12 && !(~status & 0x401) && (~oldstatus & 0x401))
561 lightrec_set_exit_flags(state, LIGHTREC_EXIT_CHECK_INTERRUPT);
98fa08a5
PC		 562 }
563}
564
565static u32 count_leading_bits(s32 data)
566{
98fa08a5
PC		 567 u32 cnt = 33;
568
de742fa0
PC		 569#ifdef __has_builtin
570#if __has_builtin(__builtin_clrsb)
571 return 1 + __builtin_clrsb(data);
572#endif
573#endif
574
98fa08a5
PC		 575 data = (data ^ (data >> 31)) << 1;
576
577 do {
578 cnt -= 1;
579 data >>= 1;
580 } while (data);
581
582 return cnt;
98fa08a5
PC		 583}
584
585static void lightrec_mtc2(struct lightrec_state *state, u8 reg, u32 data)
586{
587 switch (reg) {
588 case 15:
589 state->regs.cp2d[12] = state->regs.cp2d[13];
590 state->regs.cp2d[13] = state->regs.cp2d[14];
591 state->regs.cp2d[14] = data;
592 break;
593 case 28:
594 state->regs.cp2d[9] = (data << 7) & 0xf80;
595 state->regs.cp2d[10] = (data << 2) & 0xf80;
596 state->regs.cp2d[11] = (data >> 3) & 0xf80;
597 break;
598 case 31:
599 return;
600 case 30:
601 state->regs.cp2d[31] = count_leading_bits((s32) data);
d8b04acd
PC		 602 fallthrough;
603 default:
98fa08a5
PC		 604 state->regs.cp2d[reg] = data;
605 break;
606 }
607}
d16005f8 608
98fa08a5
PC		 609static void lightrec_ctc2(struct lightrec_state *state, u8 reg, u32 data)
610{
611 switch (reg) {
612 case 4:
613 case 12:
614 case 20:
615 case 26:
616 case 27:
617 case 29:
618 case 30:
11357fef 619 store_u16(&state->regs.cp2c[reg], data);
98fa08a5
PC		 620 break;
621 case 31:
622 data = (data & 0x7ffff000) | !!(data & 0x7f87e000) << 31;
d8b04acd
PC		 623 fallthrough;
624 default:
11357fef 625 state->regs.cp2c[reg] = data;
98fa08a5
PC		 626 break;
627 }
98fa08a5
PC		 628}
629
9259d748 630void lightrec_mtc(struct lightrec_state *state, union code op, u8 reg, u32 data)
98fa08a5 631{
fdf33147 632 if (op.i.op == OP_CP0) {
9259d748 633 lightrec_mtc0(state, reg, data);
fdf33147 634 } else {
9259d748
PC		 635 if (op.i.op == OP_LWC2 || op.r.rs != OP_CP2_BASIC_CTC2)
636 lightrec_mtc2(state, reg, data);
fdf33147 637 else
9259d748 638 lightrec_ctc2(state, reg, data);
fdf33147
PC		 639
640 if (state->ops.cop2_notify)
641 (*state->ops.cop2_notify)(state, op.opcode, data);
642 }
d16005f8
PC		 643}
644
03535202 645static void lightrec_mtc_cb(struct lightrec_state *state, u32 arg)
d16005f8 646{
03535202 647 union code op = (union code) arg;
9259d748
PC		 648 u32 data;
649 u8 reg;
650
651 if (op.i.op == OP_LWC2) {
cb72ea13 652 data = state->temp_reg;
9259d748
PC		 653 reg = op.i.rt;
654 } else {
655 data = state->regs.gpr[op.r.rt];
656 reg = op.r.rd;
657 }
22eee2ac 658
9259d748 659 lightrec_mtc(state, op, reg, data);
d16005f8
PC		 660}
661
98fa08a5 662void lightrec_rfe(struct lightrec_state *state)
d16005f8
PC		 663{
664 u32 status;
665
666 /* Read CP0 Status register (r12) */
98fa08a5 667 status = state->regs.cp0[12];
d16005f8
PC		 668
669 /* Switch the bits */
670 status = ((status & 0x3c) >> 2) | (status & ~0xf);
671
672 /* Write it back */
98fa08a5 673 lightrec_mtc0(state, 12, status);
d16005f8
PC		 674}
675
98fa08a5 676void lightrec_cp(struct lightrec_state *state, union code op)
d16005f8 677{
98fa08a5
PC		 678 if (op.i.op == OP_CP0) {
679 pr_err("Invalid CP opcode to coprocessor #0\n");
680 return;
681 }
d16005f8 682
98fa08a5 683 (*state->ops.cop2_op)(state, op.opcode);
d16005f8
PC		 684}
685
03535202 686static void lightrec_cp_cb(struct lightrec_state *state, u32 arg)
22eee2ac 687{
03535202 688 lightrec_cp(state, (union code) arg);
22eee2ac
PC		 689}
690
03535202 691static struct block * lightrec_get_block(struct lightrec_state *state, u32 pc)
d16005f8
PC		 692{
693 struct block *block = lightrec_find_block(state->block_cache, pc);
ba3814c1 694 u8 old_flags;
d16005f8 695
98fa08a5 696 if (block && lightrec_block_is_outdated(state, block)) {
f5ee77ca 697 pr_debug("Block at "PC_FMT" is outdated!\n", block->pc);
d16005f8 698
ba3814c1
PC		 699 old_flags = block_set_flags(block, BLOCK_IS_DEAD);
700 if (!(old_flags & BLOCK_IS_DEAD)) {
701 /* Make sure the recompiler isn't processing the block
702 * we'll destroy */
703 if (ENABLE_THREADED_COMPILER)
704 lightrec_recompiler_remove(state->rec, block);
705
706 lightrec_unregister_block(state->block_cache, block);
707 remove_from_code_lut(state->block_cache, block);
708 lightrec_free_block(state, block);
709 }
d16005f8 710
d16005f8
PC		 711 block = NULL;
712 }
713
714 if (!block) {
715 block = lightrec_precompile_block(state, pc);
716 if (!block) {
f5ee77ca 717 pr_err("Unable to recompile block at "PC_FMT"\n", pc);
d16005f8
PC		 718 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
719 return NULL;
720 }
721
722 lightrec_register_block(state->block_cache, block);
723 }
724
725 return block;
726}
727
728static void * get_next_block_func(struct lightrec_state *state, u32 pc)
729{
730 struct block *block;
731 bool should_recompile;
732 void *func;
d8b04acd 733 int err;
d16005f8 734
9259d748 735 do {
d8b04acd 736 func = lut_read(state, lut_offset(pc));
d16005f8 737 if (func && func != state->get_next_block)
98fa08a5 738 break;
d16005f8
PC		 739
740 block = lightrec_get_block(state, pc);
741
742 if (unlikely(!block))
98fa08a5
PC		 743 break;
744
ba3814c1
PC		 745 if (OPT_REPLACE_MEMSET &&
746 block_has_flag(block, BLOCK_IS_MEMSET)) {
98fa08a5
PC		 747 func = state->memset_func;
748 break;
749 }
d16005f8 750
ba3814c1 751 should_recompile = block_has_flag(block, BLOCK_SHOULD_RECOMPILE) &&
cb72ea13 752 !block_has_flag(block, BLOCK_NEVER_COMPILE) &&
ba3814c1 753 !block_has_flag(block, BLOCK_IS_DEAD);
d16005f8
PC		 754
755 if (unlikely(should_recompile)) {
f5ee77ca 756 pr_debug("Block at "PC_FMT" should recompile\n", pc);
d16005f8 757
d8b04acd 758 if (ENABLE_THREADED_COMPILER) {
a59e5536 759 lightrec_recompiler_add(state->rec, block);
d8b04acd
PC		 760 } else {
761 err = lightrec_compile_block(state->cstate, block);
762 if (err) {
763 state->exit_flags = LIGHTREC_EXIT_NOMEM;
764 return NULL;
765 }
766 }
d16005f8
PC		 767 }
768
769 if (ENABLE_THREADED_COMPILER && likely(!should_recompile))
98fa08a5 770 func = lightrec_recompiler_run_first_pass(state, block, &pc);
d16005f8
PC		 771 else
772 func = block->function;
773
774 if (likely(func))
98fa08a5 775 break;
d16005f8 776
ba3814c1 777 if (unlikely(block_has_flag(block, BLOCK_NEVER_COMPILE))) {
98fa08a5
PC		 778 pc = lightrec_emulate_block(state, block, pc);
779
780 } else if (!ENABLE_THREADED_COMPILER) {
781 /* Block wasn't compiled yet - run the interpreter */
ba3814c1 782 if (block_has_flag(block, BLOCK_FULLY_TAGGED))
98fa08a5
PC		 783 pr_debug("Block fully tagged, skipping first pass\n");
784 else if (ENABLE_FIRST_PASS && likely(!should_recompile))
785 pc = lightrec_emulate_block(state, block, pc);
d16005f8 786
d16005f8 787 /* Then compile it using the profiled data */
d8b04acd
PC		 788 err = lightrec_compile_block(state->cstate, block);
789 if (err) {
790 state->exit_flags = LIGHTREC_EXIT_NOMEM;
791 return NULL;
792 }
ba3814c1
PC		 793 } else if (unlikely(block_has_flag(block, BLOCK_IS_DEAD))) {
794 /*
795 * If the block is dead but has never been compiled,
796 * then its function pointer is NULL and we cannot
797 * execute the block. In that case, reap all the dead
798 * blocks now, and in the next loop we will create a
799 * new block.
800 */
801 lightrec_reaper_reap(state->reaper);
98fa08a5
PC		 802 } else {
803 lightrec_recompiler_add(state->rec, block);
d16005f8 804 }
9259d748
PC		 805 } while (state->exit_flags == LIGHTREC_EXIT_NORMAL
806 && state->current_cycle < state->target_cycle);
d16005f8 807
0e720fb1 808 state->curr_pc = pc;
98fa08a5 809 return func;
d16005f8
PC		 810}
811
d8b04acd
PC		 812static void * lightrec_alloc_code(struct lightrec_state *state, size_t size)
813{
814 void *code;
815
816 if (ENABLE_THREADED_COMPILER)
817 lightrec_code_alloc_lock(state);
818
819 code = tlsf_malloc(state->tlsf, size);
820
821 if (ENABLE_THREADED_COMPILER)
822 lightrec_code_alloc_unlock(state);
823
824 return code;
825}
826
827static void lightrec_realloc_code(struct lightrec_state *state,
828 void *ptr, size_t size)
829{
830 /* NOTE: 'size' MUST be smaller than the size specified during
831 * the allocation. */
832
833 if (ENABLE_THREADED_COMPILER)
834 lightrec_code_alloc_lock(state);
835
836 tlsf_realloc(state->tlsf, ptr, size);
837
838 if (ENABLE_THREADED_COMPILER)
839 lightrec_code_alloc_unlock(state);
840}
841
842static void lightrec_free_code(struct lightrec_state *state, void *ptr)
843{
844 if (ENABLE_THREADED_COMPILER)
845 lightrec_code_alloc_lock(state);
846
847 tlsf_free(state->tlsf, ptr);
848
849 if (ENABLE_THREADED_COMPILER)
850 lightrec_code_alloc_unlock(state);
851}
852
cb72ea13
PC		 853static char lightning_code_data[0x80000];
854
02487de7 855static void * lightrec_emit_code(struct lightrec_state *state,
d8b04acd 856 const struct block *block,
02487de7
PC		 857 jit_state_t *_jit, unsigned int *size)
858{
859 bool has_code_buffer = ENABLE_CODE_BUFFER && state->tlsf;
860 jit_word_t code_size, new_code_size;
861 void *code;
862
863 jit_realize();
864
cb72ea13
PC		 865 if (ENABLE_DISASSEMBLER)
866 jit_set_data(lightning_code_data, sizeof(lightning_code_data), 0);
867 else
02487de7
PC		 868 jit_set_data(NULL, 0, JIT_DISABLE_DATA | JIT_DISABLE_NOTE);
869
870 if (has_code_buffer) {
871 jit_get_code(&code_size);
d8b04acd
PC		 872 code = lightrec_alloc_code(state, (size_t) code_size);
873
874 if (!code) {
875 if (ENABLE_THREADED_COMPILER) {
876 /* If we're using the threaded compiler, return
877 * an allocation error here. The threaded
878 * compiler will then empty its job queue and
879 * request a code flush using the reaper. */
880 return NULL;
881 }
882
883 /* Remove outdated blocks, and try again */
884 lightrec_remove_outdated_blocks(state->block_cache, block);
885
886 pr_debug("Re-try to alloc %zu bytes...\n", code_size);
887
888 code = lightrec_alloc_code(state, code_size);
889 if (!code) {
890 pr_err("Could not alloc even after removing old blocks!\n");
891 return NULL;
892 }
893 }
02487de7
PC		 894
895 jit_set_code(code, code_size);
896 }
897
898 code = jit_emit();
899
900 jit_get_code(&new_code_size);
901 lightrec_register(MEM_FOR_CODE, new_code_size);
902
903 if (has_code_buffer) {
d8b04acd 904 lightrec_realloc_code(state, code, (size_t) new_code_size);
02487de7
PC		 905
906 pr_debug("Creating code block at address 0x%" PRIxPTR ", "
907 "code size: %" PRIuPTR " new: %" PRIuPTR "\n",
908 (uintptr_t) code, code_size, new_code_size);
909 }
910
911 *size = (unsigned int) new_code_size;
912
9259d748
PC		 913 if (state->ops.code_inv)
914 state->ops.code_inv(code, new_code_size);
915
02487de7
PC		 916 return code;
917}
918
98fa08a5 919static struct block * generate_wrapper(struct lightrec_state *state)
d16005f8
PC		 920{
921 struct block *block;
922 jit_state_t *_jit;
923 unsigned int i;
fd58fa32 924 jit_node_t *addr[C_WRAPPERS_COUNT - 1];
ba3814c1 925 jit_node_t *to_end[C_WRAPPERS_COUNT - 1];
cb72ea13
PC		 926 u8 tmp = JIT_R1;
927
928#ifdef __sh__
929 /* On SH, GBR-relative loads target the r0 register.
930 * Use it as the temporary register to factorize the move to
931 * JIT_R1. */
932 if (LIGHTREC_REG_STATE == _GBR)
933 tmp = _R0;
934#endif
d16005f8
PC		 935
936 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
937 if (!block)
938 goto err_no_mem;
939
940 _jit = jit_new_state();
941 if (!_jit)
942 goto err_free_block;
943
944 jit_name("RW wrapper");
945 jit_note(__FILE__, __LINE__);
946
947 /* Wrapper entry point */
948 jit_prolog();
fd58fa32
PC		 949 jit_tramp(256);
950
ba3814c1 951 /* Add entry points */
fd58fa32 952 for (i = C_WRAPPERS_COUNT - 1; i > 0; i--) {
cb72ea13 953 jit_ldxi(tmp, LIGHTREC_REG_STATE,
ba3814c1
PC		 954 offsetof(struct lightrec_state, c_wrappers[i]));
955 to_end[i - 1] = jit_b();
fd58fa32
PC		 956 addr[i - 1] = jit_indirect();
957 }
958
cb72ea13 959 jit_ldxi(tmp, LIGHTREC_REG_STATE,
ba3814c1
PC		 960 offsetof(struct lightrec_state, c_wrappers[0]));
961
962 for (i = 0; i < C_WRAPPERS_COUNT - 1; i++)
963 jit_patch(to_end[i]);
cb72ea13 964 jit_movr(JIT_R1, tmp);
ba3814c1 965
fd58fa32
PC		 966 jit_epilog();
967 jit_prolog();
d16005f8 968
fd58fa32 969 /* Save all temporaries on stack */
ba3814c1
PC		 970 for (i = 0; i < NUM_TEMPS; i++) {
971 if (i + FIRST_TEMP != 1) {
972 jit_stxi(offsetof(struct lightrec_state, wrapper_regs[i]),
973 LIGHTREC_REG_STATE, JIT_R(i + FIRST_TEMP));
974 }
975 }
d16005f8 976
ba3814c1 977 jit_getarg(JIT_R2, jit_arg());
03535202 978
ba3814c1
PC		 979 jit_prepare();
980 jit_pushargr(LIGHTREC_REG_STATE);
981 jit_pushargr(JIT_R2);
d16005f8 982
ba3814c1
PC		 983 jit_ldxi_ui(JIT_R2, LIGHTREC_REG_STATE,
984 offsetof(struct lightrec_state, target_cycle));
d16005f8 985
ba3814c1
PC		 986 /* state->current_cycle = state->target_cycle - delta; */
987 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, LIGHTREC_REG_CYCLE);
988 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
989 LIGHTREC_REG_STATE, LIGHTREC_REG_CYCLE);
d16005f8 990
ba3814c1
PC		 991 /* Call the wrapper function */
992 jit_finishr(JIT_R1);
d16005f8 993
ba3814c1
PC		 994 /* delta = state->target_cycle - state->current_cycle */;
995 jit_ldxi_ui(LIGHTREC_REG_CYCLE, LIGHTREC_REG_STATE,
996 offsetof(struct lightrec_state, current_cycle));
997 jit_ldxi_ui(JIT_R1, LIGHTREC_REG_STATE,
998 offsetof(struct lightrec_state, target_cycle));
999 jit_subr(LIGHTREC_REG_CYCLE, JIT_R1, LIGHTREC_REG_CYCLE);
fd58fa32 1000
ba3814c1
PC		 1001 /* Restore temporaries from stack */
1002 for (i = 0; i < NUM_TEMPS; i++) {
1003 if (i + FIRST_TEMP != 1) {
1004 jit_ldxi(JIT_R(i + FIRST_TEMP), LIGHTREC_REG_STATE,
1005 offsetof(struct lightrec_state, wrapper_regs[i]));
1006 }
1007 }
d16005f8 1008
ba3814c1 1009 jit_ret();
d16005f8
PC		 1010 jit_epilog();
1011
d16005f8 1012 block->_jit = _jit;
d16005f8 1013 block->opcode_list = NULL;
ba3814c1 1014 block->flags = BLOCK_NO_OPCODE_LIST;
d16005f8
PC		 1015 block->nb_ops = 0;
1016
d8b04acd 1017 block->function = lightrec_emit_code(state, block, _jit,
02487de7
PC		 1018 &block->code_size);
1019 if (!block->function)
1020 goto err_free_block;
1021
fd58fa32
PC		 1022 state->wrappers_eps[C_WRAPPERS_COUNT - 1] = block->function;
1023
1024 for (i = 0; i < C_WRAPPERS_COUNT - 1; i++)
1025 state->wrappers_eps[i] = jit_address(addr[i]);
1026
d16005f8
PC		 1027 if (ENABLE_DISASSEMBLER) {
1028 pr_debug("Wrapper block:\n");
1029 jit_disassemble();
1030 }
1031
1032 jit_clear_state();
1033 return block;
1034
1035err_free_block:
1036 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
1037err_no_mem:
1038 pr_err("Unable to compile wrapper: Out of memory\n");
1039 return NULL;
1040}
1041
98fa08a5
PC		 1042static u32 lightrec_memset(struct lightrec_state *state)
1043{
1044 u32 kunseg_pc = kunseg(state->regs.gpr[4]);
1045 void *host;
1046 const struct lightrec_mem_map *map = lightrec_get_map(state, &host, kunseg_pc);
1047 u32 length = state->regs.gpr[5] * 4;
1048
1049 if (!map) {
f5ee77ca
PC		 1050 pr_err("Unable to find memory map for memset target address "PC_FMT"\n",
1051 kunseg_pc);
98fa08a5
PC		 1052 return 0;
1053 }
1054
f5ee77ca 1055 pr_debug("Calling host memset, "PC_FMT" (host address 0x%"PRIxPTR") for %u bytes\n",
98fa08a5
PC		 1056 kunseg_pc, (uintptr_t)host, length);
1057 memset(host, 0, length);
1058
684432ad 1059 if (!(state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY))
98fa08a5
PC		 1060 lightrec_invalidate_map(state, map, kunseg_pc, length);
1061
1062 /* Rough estimation of the number of cycles consumed */
1063 return 8 + 5 * (length + 3 / 4);
1064}
1065
cb72ea13
PC		 1066static u32 lightrec_check_load_delay(struct lightrec_state *state, u32 pc, u8 reg)
1067{
1068 struct block *block;
1069 union code first_op;
1070
1071 first_op = lightrec_read_opcode(state, pc);
1072
1073 if (likely(!opcode_reads_register(first_op, reg))) {
1074 state->regs.gpr[reg] = state->temp_reg;
1075 } else {
1076 block = lightrec_get_block(state, pc);
1077 if (unlikely(!block)) {
f5ee77ca 1078 pr_err("Unable to get block at "PC_FMT"\n", pc);
cb72ea13
PC		 1079 lightrec_set_exit_flags(state, LIGHTREC_EXIT_SEGFAULT);
1080 pc = 0;
1081 } else {
1082 pc = lightrec_handle_load_delay(state, block, pc, reg);
1083 }
1084 }
1085
1086 return pc;
1087}
1088
1089static void update_cycle_counter_before_c(jit_state_t *_jit)
1090{
1091 /* update state->current_cycle */
1092 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
1093 offsetof(struct lightrec_state, target_cycle));
1094 jit_subr(JIT_R1, JIT_R2, LIGHTREC_REG_CYCLE);
1095 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
1096 LIGHTREC_REG_STATE, JIT_R1);
1097}
1098
1099static void update_cycle_counter_after_c(jit_state_t *_jit)
1100{
1101 /* Recalc the delta */
1102 jit_ldxi_i(JIT_R1, LIGHTREC_REG_STATE,
1103 offsetof(struct lightrec_state, current_cycle));
1104 jit_ldxi_i(JIT_R2, LIGHTREC_REG_STATE,
1105 offsetof(struct lightrec_state, target_cycle));
1106 jit_subr(LIGHTREC_REG_CYCLE, JIT_R2, JIT_R1);
1107}
1108
0e720fb1
PC		 1109static void sync_next_pc(jit_state_t *_jit)
1110{
1111 if (lightrec_store_next_pc()) {
684432ad
PC		 1112 jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE,
1113 offsetof(struct lightrec_state, next_pc));
0e720fb1
PC		 1114 }
1115}
1116
d16005f8
PC		 1117static struct block * generate_dispatcher(struct lightrec_state *state)
1118{
1119 struct block *block;
1120 jit_state_t *_jit;
cb72ea13 1121 jit_node_t *to_end, *loop, *addr, *addr2, *addr3, *addr4, *addr5, *jmp, *jmp2;
d16005f8 1122 unsigned int i;
02487de7 1123 u32 offset;
d16005f8
PC		 1124
1125 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
1126 if (!block)
1127 goto err_no_mem;
1128
1129 _jit = jit_new_state();
1130 if (!_jit)
1131 goto err_free_block;
1132
1133 jit_name("dispatcher");
1134 jit_note(__FILE__, __LINE__);
1135
1136 jit_prolog();
1137 jit_frame(256);
1138
9259d748
PC		 1139 jit_getarg(LIGHTREC_REG_STATE, jit_arg());
1140 jit_getarg(JIT_V0, jit_arg());
ba3814c1 1141 jit_getarg(JIT_V1, jit_arg());
d16005f8 1142 jit_getarg_i(LIGHTREC_REG_CYCLE, jit_arg());
d16005f8
PC		 1143
1144 /* Force all callee-saved registers to be pushed on the stack */
1145 for (i = 0; i < NUM_REGS; i++)
ba3814c1 1146 jit_movr(JIT_V(i + FIRST_REG), JIT_V(i + FIRST_REG));
d16005f8 1147
d16005f8
PC		 1148 loop = jit_label();
1149
1150 /* Call the block's code */
ba3814c1 1151 jit_jmpr(JIT_V1);
d16005f8 1152
98fa08a5
PC		 1153 if (OPT_REPLACE_MEMSET) {
1154 /* Blocks will jump here when they need to call
1155 * lightrec_memset() */
1156 addr3 = jit_indirect();
1157
ba3814c1
PC		 1158 jit_movr(JIT_V1, LIGHTREC_REG_CYCLE);
1159
98fa08a5
PC		 1160 jit_prepare();
1161 jit_pushargr(LIGHTREC_REG_STATE);
cb72ea13 1162
98fa08a5 1163 jit_finishi(lightrec_memset);
cb72ea13 1164 jit_retval(LIGHTREC_REG_CYCLE);
98fa08a5
PC		 1165
1166 jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE,
1167 offsetof(struct lightrec_state, regs.gpr[31]));
ba3814c1 1168 jit_subr(LIGHTREC_REG_CYCLE, JIT_V1, LIGHTREC_REG_CYCLE);
cb72ea13
PC		 1169
1170 if (OPT_DETECT_IMPOSSIBLE_BRANCHES || OPT_HANDLE_LOAD_DELAYS)
1171 jmp = jit_b();
1172 }
1173
1174 if (OPT_DETECT_IMPOSSIBLE_BRANCHES) {
1175 /* Blocks will jump here when they reach a branch that should
1176 * be executed with the interpreter, passing the branch's PC
1177 * in JIT_V0 and the address of the block in JIT_V1. */
1178 addr4 = jit_indirect();
1179
0e720fb1 1180 sync_next_pc(_jit);
cb72ea13
PC		 1181 update_cycle_counter_before_c(_jit);
1182
1183 jit_prepare();
1184 jit_pushargr(LIGHTREC_REG_STATE);
1185 jit_pushargr(JIT_V1);
1186 jit_pushargr(JIT_V0);
1187 jit_finishi(lightrec_emulate_block);
1188
1189 jit_retval(JIT_V0);
1190
1191 update_cycle_counter_after_c(_jit);
1192
1193 if (OPT_HANDLE_LOAD_DELAYS)
1194 jmp2 = jit_b();
1195
1196 }
1197
1198 if (OPT_HANDLE_LOAD_DELAYS) {
1199 /* Blocks will jump here when they reach a branch with a load
1200 * opcode in its delay slot. The delay slot has already been
1201 * executed; the load value is in (state->temp_reg), and the
1202 * register number is in JIT_V1.
1203 * Jump to a C function which will evaluate the branch target's
1204 * first opcode, to make sure that it does not read the register
1205 * in question; and if it does, handle it accordingly. */
1206 addr5 = jit_indirect();
1207
0e720fb1 1208 sync_next_pc(_jit);
cb72ea13
PC		 1209 update_cycle_counter_before_c(_jit);
1210
1211 jit_prepare();
1212 jit_pushargr(LIGHTREC_REG_STATE);
1213 jit_pushargr(JIT_V0);
1214 jit_pushargr(JIT_V1);
1215 jit_finishi(lightrec_check_load_delay);
1216
1217 jit_retval(JIT_V0);
1218
1219 update_cycle_counter_after_c(_jit);
cb72ea13
PC		 1220 }
1221
0e720fb1
PC		 1222 /* The block will jump here, with the number of cycles remaining in
1223 * LIGHTREC_REG_CYCLE */
1224 addr2 = jit_indirect();
1225
1226 sync_next_pc(_jit);
1227
1228 if (OPT_HANDLE_LOAD_DELAYS && OPT_DETECT_IMPOSSIBLE_BRANCHES)
1229 jit_patch(jmp2);
1230
cb72ea13
PC		 1231 if (OPT_REPLACE_MEMSET
1232 && (OPT_DETECT_IMPOSSIBLE_BRANCHES || OPT_HANDLE_LOAD_DELAYS)) {
1233 jit_patch(jmp);
98fa08a5
PC		 1234 }
1235
0e720fb1
PC		 1236 /* Store back the next PC to the lightrec_state structure */
1237 offset = offsetof(struct lightrec_state, curr_pc);
98fa08a5
PC		 1238 jit_stxi_i(offset, LIGHTREC_REG_STATE, JIT_V0);
1239
d16005f8
PC		 1240 /* Jump to end if state->target_cycle < state->current_cycle */
1241 to_end = jit_blei(LIGHTREC_REG_CYCLE, 0);
1242
1243 /* Convert next PC to KUNSEG and avoid mirrors */
ba3814c1
PC		 1244 jit_andi(JIT_V1, JIT_V0, 0x10000000 | (RAM_SIZE - 1));
1245 jit_rshi_u(JIT_R1, JIT_V1, 28);
02487de7
PC		 1246 jit_andi(JIT_R2, JIT_V0, BIOS_SIZE - 1);
1247 jit_addi(JIT_R2, JIT_R2, RAM_SIZE);
ba3814c1 1248 jit_movnr(JIT_V1, JIT_R2, JIT_R1);
02487de7
PC		 1249
1250 /* If possible, use the code LUT */
1251 if (!lut_is_32bit(state))
ba3814c1 1252 jit_lshi(JIT_V1, JIT_V1, 1);
cb72ea13 1253 jit_add_state(JIT_V1, JIT_V1);
02487de7
PC		 1254
1255 offset = offsetof(struct lightrec_state, code_lut);
1256 if (lut_is_32bit(state))
ba3814c1 1257 jit_ldxi_ui(JIT_V1, JIT_V1, offset);
02487de7 1258 else
ba3814c1 1259 jit_ldxi(JIT_V1, JIT_V1, offset);
d16005f8
PC		 1260
1261 /* If we get non-NULL, loop */
ba3814c1
PC		 1262 jit_patch_at(jit_bnei(JIT_V1, 0), loop);
1263
1264 /* The code LUT will be set to this address when the block at the target
1265 * PC has been preprocessed but not yet compiled by the threaded
1266 * recompiler */
1267 addr = jit_indirect();
d16005f8
PC		 1268
1269 /* Slow path: call C function get_next_block_func() */
d16005f8 1270
98fa08a5 1271 if (ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES) {
d16005f8 1272 /* We may call the interpreter - update state->current_cycle */
cb72ea13 1273 update_cycle_counter_before_c(_jit);
d16005f8
PC		 1274 }
1275
d16005f8
PC		 1276 jit_prepare();
1277 jit_pushargr(LIGHTREC_REG_STATE);
1278 jit_pushargr(JIT_V0);
ba3814c1
PC		 1279
1280 /* Save the cycles register if needed */
1281 if (!(ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES))
1282 jit_movr(JIT_V0, LIGHTREC_REG_CYCLE);
1283
1284 /* Get the next block */
d16005f8 1285 jit_finishi(&get_next_block_func);
ba3814c1 1286 jit_retval(JIT_V1);
d16005f8 1287
98fa08a5 1288 if (ENABLE_FIRST_PASS || OPT_DETECT_IMPOSSIBLE_BRANCHES) {
d16005f8
PC		 1289 /* The interpreter may have updated state->current_cycle and
1290 * state->target_cycle - recalc the delta */
cb72ea13 1291 update_cycle_counter_after_c(_jit);
ba3814c1
PC		 1292 } else {
1293 jit_movr(LIGHTREC_REG_CYCLE, JIT_V0);
d16005f8
PC		 1294 }
1295
9259d748
PC		 1296 /* Reset JIT_V0 to the next PC */
1297 jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE,
0e720fb1 1298 offsetof(struct lightrec_state, curr_pc));
9259d748 1299
d16005f8 1300 /* If we get non-NULL, loop */
ba3814c1 1301 jit_patch_at(jit_bnei(JIT_V1, 0), loop);
d16005f8 1302
d16005f8
PC		 1303 /* When exiting, the recompiled code will jump to that address */
1304 jit_note(__FILE__, __LINE__);
1305 jit_patch(to_end);
1306
d16005f8
PC		 1307 jit_retr(LIGHTREC_REG_CYCLE);
1308 jit_epilog();
1309
d16005f8 1310 block->_jit = _jit;
d16005f8 1311 block->opcode_list = NULL;
ba3814c1 1312 block->flags = BLOCK_NO_OPCODE_LIST;
d16005f8
PC		 1313 block->nb_ops = 0;
1314
d8b04acd 1315 block->function = lightrec_emit_code(state, block, _jit,
02487de7
PC		 1316 &block->code_size);
1317 if (!block->function)
1318 goto err_free_block;
d16005f8
PC		 1319
1320 state->eob_wrapper_func = jit_address(addr2);
cb72ea13
PC		 1321 if (OPT_DETECT_IMPOSSIBLE_BRANCHES)
1322 state->interpreter_func = jit_address(addr4);
1323 if (OPT_HANDLE_LOAD_DELAYS)
1324 state->ds_check_func = jit_address(addr5);
98fa08a5
PC		 1325 if (OPT_REPLACE_MEMSET)
1326 state->memset_func = jit_address(addr3);
d16005f8
PC		 1327 state->get_next_block = jit_address(addr);
1328
1329 if (ENABLE_DISASSEMBLER) {
1330 pr_debug("Dispatcher block:\n");
1331 jit_disassemble();
1332 }
1333
1334 /* We're done! */
1335 jit_clear_state();
1336 return block;
1337
1338err_free_block:
1339 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
1340err_no_mem:
1341 pr_err("Unable to compile dispatcher: Out of memory\n");
1342 return NULL;
1343}
1344
1345union code lightrec_read_opcode(struct lightrec_state *state, u32 pc)
1346{
fd58fa32 1347 void *host = NULL;
d16005f8 1348
98fa08a5 1349 lightrec_get_map(state, &host, kunseg(pc));
d16005f8 1350
98fa08a5 1351 const u32 *code = (u32 *)host;
02487de7 1352 return (union code) LE32TOH(*code);
98fa08a5 1353}
d16005f8 1354
684432ad
PC		 1355unsigned int lightrec_cycles_of_opcode(const struct lightrec_state *state,
1356 union code code)
98fa08a5 1357{
684432ad 1358 return state->cycles_per_op;
98fa08a5 1359}
d16005f8 1360
ba3814c1 1361void lightrec_free_opcode_list(struct lightrec_state *state, struct opcode *ops)
98fa08a5 1362{
ba3814c1
PC		 1363 struct opcode_list *list = container_of(ops, struct opcode_list, ops);
1364
98fa08a5 1365 lightrec_free(state, MEM_FOR_IR,
ba3814c1
PC		 1366 sizeof(*list) + list->nb_ops * sizeof(struct opcode),
1367 list);
98fa08a5
PC		 1368}
1369
1370static unsigned int lightrec_get_mips_block_len(const u32 *src)
1371{
1372 unsigned int i;
1373 union code c;
1374
1375 for (i = 1; ; i++) {
1376 c.opcode = LE32TOH(*src++);
1377
1378 if (is_syscall(c))
1379 return i;
1380
1381 if (is_unconditional_jump(c))
1382 return i + 1;
1383 }
1384}
1385
1386static struct opcode * lightrec_disassemble(struct lightrec_state *state,
1387 const u32 *src, unsigned int *len)
1388{
ba3814c1 1389 struct opcode_list *list;
98fa08a5
PC		 1390 unsigned int i, length;
1391
1392 length = lightrec_get_mips_block_len(src);
1393
ba3814c1
PC		 1394 list = lightrec_malloc(state, MEM_FOR_IR,
1395 sizeof(*list) + sizeof(struct opcode) * length);
98fa08a5
PC		 1396 if (!list) {
1397 pr_err("Unable to allocate memory\n");
1398 return NULL;
1399 }
1400
ba3814c1
PC		 1401 list->nb_ops = (u16) length;
1402
98fa08a5 1403 for (i = 0; i < length; i++) {
ba3814c1
PC		 1404 list->ops[i].opcode = LE32TOH(src[i]);
1405 list->ops[i].flags = 0;
98fa08a5
PC		 1406 }
1407
1408 *len = length * sizeof(u32);
1409
ba3814c1 1410 return list->ops;
d16005f8
PC		 1411}
1412
1413static struct block * lightrec_precompile_block(struct lightrec_state *state,
1414 u32 pc)
1415{
1416 struct opcode *list;
1417 struct block *block;
ba3814c1 1418 void *host, *addr;
98fa08a5
PC		 1419 const struct lightrec_mem_map *map = lightrec_get_map(state, &host, kunseg(pc));
1420 const u32 *code = (u32 *) host;
d16005f8 1421 unsigned int length;
98fa08a5 1422 bool fully_tagged;
ba3814c1 1423 u8 block_flags = 0;
d16005f8
PC		 1424
1425 if (!map)
1426 return NULL;
1427
d16005f8
PC		 1428 block = lightrec_malloc(state, MEM_FOR_IR, sizeof(*block));
1429 if (!block) {
1430 pr_err("Unable to recompile block: Out of memory\n");
1431 return NULL;
1432 }
1433
1434 list = lightrec_disassemble(state, code, &length);
1435 if (!list) {
1436 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
1437 return NULL;
1438 }
1439
1440 block->pc = pc;
d16005f8
PC		 1441 block->_jit = NULL;
1442 block->function = NULL;
1443 block->opcode_list = list;
98fa08a5 1444 block->code = code;
d16005f8
PC		 1445 block->next = NULL;
1446 block->flags = 0;
1447 block->code_size = 0;
d8b04acd 1448 block->precompile_date = state->current_cycle;
d16005f8
PC		 1449 block->nb_ops = length / sizeof(u32);
1450
98fa08a5 1451 lightrec_optimize(state, block);
d16005f8
PC		 1452
1453 length = block->nb_ops * sizeof(u32);
1454
1455 lightrec_register(MEM_FOR_MIPS_CODE, length);
1456
1457 if (ENABLE_DISASSEMBLER) {
98fa08a5
PC		 1458 pr_debug("Disassembled block at PC: 0x%08x\n", block->pc);
1459 lightrec_print_disassembly(block, code);
d16005f8
PC		 1460 }
1461
98fa08a5 1462 pr_debug("Block size: %hu opcodes\n", block->nb_ops);
d16005f8 1463
98fa08a5
PC		 1464 fully_tagged = lightrec_block_is_fully_tagged(block);
1465 if (fully_tagged)
ba3814c1 1466 block_flags |= BLOCK_FULLY_TAGGED;
98fa08a5 1467
ba3814c1
PC		 1468 if (block_flags)
1469 block_set_flags(block, block_flags);
98fa08a5 1470
d16005f8
PC		 1471 block->hash = lightrec_calculate_block_hash(block);
1472
ba3814c1
PC		 1473 if (OPT_REPLACE_MEMSET && block_has_flag(block, BLOCK_IS_MEMSET))
1474 addr = state->memset_func;
1475 else
1476 addr = state->get_next_block;
1477 lut_write(state, lut_offset(pc), addr);
1478
cb72ea13 1479 pr_debug("Blocks created: %u\n", ++state->nb_precompile);
a59e5536 1480
d16005f8
PC		 1481 return block;
1482}
1483
98fa08a5 1484static bool lightrec_block_is_fully_tagged(const struct block *block)
d16005f8 1485{
98fa08a5
PC		 1486 const struct opcode *op;
1487 unsigned int i;
1488
1489 for (i = 0; i < block->nb_ops; i++) {
1490 op = &block->opcode_list[i];
d16005f8 1491
cb72ea13
PC		 1492 /* If we have one branch that must be emulated, we cannot trash
1493 * the opcode list. */
1494 if (should_emulate(op))
1495 return false;
1496
1497 /* Check all loads/stores of the opcode list and mark the
d16005f8
PC		 1498 * block as fully compiled if they all have been tagged. */
1499 switch (op->c.i.op) {
1500 case OP_LB:
1501 case OP_LH:
1502 case OP_LWL:
1503 case OP_LW:
1504 case OP_LBU:
1505 case OP_LHU:
1506 case OP_LWR:
1507 case OP_SB:
1508 case OP_SH:
1509 case OP_SWL:
1510 case OP_SW:
1511 case OP_SWR:
1512 case OP_LWC2:
1513 case OP_SWC2:
5459088b
PC		 1514 case OP_META_LWU:
1515 case OP_META_SWU:
22eee2ac 1516 if (!LIGHTREC_FLAGS_GET_IO_MODE(op->flags))
d16005f8 1517 return false;
d8b04acd
PC		 1518 fallthrough;
1519 default:
d16005f8
PC		 1520 continue;
1521 }
1522 }
1523
1524 return true;
1525}
1526
98fa08a5 1527static void lightrec_reap_block(struct lightrec_state *state, void *data)
a59e5536 1528{
1529 struct block *block = data;
1530
f5ee77ca 1531 pr_debug("Reap dead block at "PC_FMT"\n", block->pc);
98fa08a5
PC		 1532 lightrec_unregister_block(state->block_cache, block);
1533 lightrec_free_block(state, block);
a59e5536 1534}
1535
98fa08a5 1536static void lightrec_reap_jit(struct lightrec_state *state, void *data)
a59e5536 1537{
1538 _jit_destroy_state(data);
1539}
1540
02487de7
PC		 1541static void lightrec_free_function(struct lightrec_state *state, void *fn)
1542{
1543 if (ENABLE_CODE_BUFFER && state->tlsf) {
1544 pr_debug("Freeing code block at 0x%" PRIxPTR "\n", (uintptr_t) fn);
d8b04acd 1545 lightrec_free_code(state, fn);
02487de7
PC		 1546 }
1547}
1548
1549static void lightrec_reap_function(struct lightrec_state *state, void *data)
1550{
1551 lightrec_free_function(state, data);
1552}
1553
ba3814c1
PC		 1554static void lightrec_reap_opcode_list(struct lightrec_state *state, void *data)
1555{
1556 lightrec_free_opcode_list(state, data);
1557}
1558
98fa08a5
PC		 1559int lightrec_compile_block(struct lightrec_cstate *cstate,
1560 struct block *block)
d16005f8 1561{
98fa08a5 1562 struct lightrec_state *state = cstate->state;
a59e5536 1563 struct lightrec_branch_target *target;
ba3814c1 1564 bool fully_tagged = false;
a59e5536 1565 struct block *block2;
d16005f8 1566 struct opcode *elm;
a59e5536 1567 jit_state_t *_jit, *oldjit;
d16005f8
PC		 1568 jit_node_t *start_of_block;
1569 bool skip_next = false;
d8b04acd 1570 void *old_fn, *new_fn;
ba3814c1 1571 size_t old_code_size;
d16005f8 1572 unsigned int i, j;
ba3814c1 1573 u8 old_flags;
98fa08a5 1574 u32 offset;
d16005f8
PC		 1575
1576 fully_tagged = lightrec_block_is_fully_tagged(block);
1577 if (fully_tagged)
ba3814c1 1578 block_set_flags(block, BLOCK_FULLY_TAGGED);
d16005f8
PC		 1579
1580 _jit = jit_new_state();
1581 if (!_jit)
1582 return -ENOMEM;
1583
a59e5536 1584 oldjit = block->_jit;
02487de7 1585 old_fn = block->function;
ba3814c1 1586 old_code_size = block->code_size;
d16005f8
PC		 1587 block->_jit = _jit;
1588
98fa08a5 1589 lightrec_regcache_reset(cstate->reg_cache);
684432ad
PC		 1590
1591 if (OPT_PRELOAD_PC && (block->flags & BLOCK_PRELOAD_PC))
1592 lightrec_preload_pc(cstate->reg_cache, _jit);
9259d748 1593
98fa08a5 1594 cstate->cycles = 0;
98fa08a5
PC		 1595 cstate->nb_local_branches = 0;
1596 cstate->nb_targets = 0;
cb72ea13 1597 cstate->no_load_delay = false;
d16005f8
PC		 1598
1599 jit_prolog();
1600 jit_tramp(256);
1601
1602 start_of_block = jit_label();
1603
98fa08a5
PC		 1604 for (i = 0; i < block->nb_ops; i++) {
1605 elm = &block->opcode_list[i];
d16005f8
PC		 1606
1607 if (skip_next) {
1608 skip_next = false;
1609 continue;
1610 }
1611
98fa08a5 1612 if (should_emulate(elm)) {
d16005f8 1613 pr_debug("Branch at offset 0x%x will be emulated\n",
98fa08a5
PC		 1614 i << 2);
1615
cb72ea13 1616 lightrec_emit_jump_to_interpreter(cstate, block, i);
03535202 1617 skip_next = !op_flag_no_ds(elm->flags);
98fa08a5
PC		 1618 } else {
1619 lightrec_rec_opcode(cstate, block, i);
03535202 1620 skip_next = !op_flag_no_ds(elm->flags) && has_delay_slot(elm->c);
d16005f8
PC		 1621#if _WIN32
1622 /* FIXME: GNU Lightning on Windows seems to use our
1623 * mapped registers as temporaries. Until the actual bug
1624 * is found and fixed, unconditionally mark our
1625 * registers as live here. */
98fa08a5 1626 lightrec_regcache_mark_live(cstate->reg_cache, _jit);
d16005f8
PC		 1627#endif
1628 }
03535202 1629
684432ad 1630 cstate->cycles += lightrec_cycles_of_opcode(state, elm->c);
d16005f8
PC		 1631 }
1632
98fa08a5
PC		 1633 for (i = 0; i < cstate->nb_local_branches; i++) {
1634 struct lightrec_branch *branch = &cstate->local_branches[i];
d16005f8
PC		 1635
1636 pr_debug("Patch local branch to offset 0x%x\n",
1637 branch->target << 2);
1638
1639 if (branch->target == 0) {
1640 jit_patch_at(branch->branch, start_of_block);
1641 continue;
1642 }
1643
98fa08a5
PC		 1644 for (j = 0; j < cstate->nb_targets; j++) {
1645 if (cstate->targets[j].offset == branch->target) {
d16005f8 1646 jit_patch_at(branch->branch,
98fa08a5 1647 cstate->targets[j].label);
d16005f8
PC		 1648 break;
1649 }
1650 }
1651
98fa08a5 1652 if (j == cstate->nb_targets)
d16005f8
PC		 1653 pr_err("Unable to find branch target\n");
1654 }
1655
d16005f8
PC		 1656 jit_ret();
1657 jit_epilog();
1658
d8b04acd
PC		 1659 new_fn = lightrec_emit_code(state, block, _jit, &block->code_size);
1660 if (!new_fn) {
1661 if (!ENABLE_THREADED_COMPILER)
1662 pr_err("Unable to compile block!\n");
1663 block->_jit = oldjit;
ba3814c1 1664 jit_clear_state();
d8b04acd
PC		 1665 _jit_destroy_state(_jit);
1666 return -ENOMEM;
02487de7
PC		 1667 }
1668
ba3814c1
PC		 1669 /* Pause the reaper, because lightrec_reset_lut_offset() may try to set
1670 * the old block->function pointer to the code LUT. */
1671 if (ENABLE_THREADED_COMPILER)
1672 lightrec_reaper_pause(state->reaper);
1673
d8b04acd 1674 block->function = new_fn;
ba3814c1 1675 block_clear_flags(block, BLOCK_SHOULD_RECOMPILE);
d16005f8
PC		 1676
1677 /* Add compiled function to the LUT */
02487de7 1678 lut_write(state, lut_offset(block->pc), block->function);
d16005f8 1679
ba3814c1
PC		 1680 if (ENABLE_THREADED_COMPILER)
1681 lightrec_reaper_continue(state->reaper);
a59e5536 1682
1683 /* Detect old blocks that have been covered by the new one */
98fa08a5
PC		 1684 for (i = 0; i < cstate->nb_targets; i++) {
1685 target = &cstate->targets[i];
a59e5536 1686
1687 if (!target->offset)
1688 continue;
1689
1690 offset = block->pc + target->offset * sizeof(u32);
ba3814c1
PC		 1691
1692 /* Pause the reaper while we search for the block until we set
1693 * the BLOCK_IS_DEAD flag, otherwise the block may be removed
1694 * under our feet. */
1695 if (ENABLE_THREADED_COMPILER)
1696 lightrec_reaper_pause(state->reaper);
1697
a59e5536 1698 block2 = lightrec_find_block(state->block_cache, offset);
1699 if (block2) {
1700 /* No need to check if block2 is compilable - it must
1701 * be, otherwise block wouldn't be compilable either */
1702
98fa08a5
PC		 1703 /* Set the "block dead" flag to prevent the dynarec from
1704 * recompiling this block */
ba3814c1
PC		 1705 old_flags = block_set_flags(block2, BLOCK_IS_DEAD);
1706 }
1707
1708 if (ENABLE_THREADED_COMPILER) {
1709 lightrec_reaper_continue(state->reaper);
a59e5536 1710
98fa08a5
PC		 1711 /* If block2 was pending for compilation, cancel it.
1712 * If it's being compiled right now, wait until it
1713 * finishes. */
ba3814c1 1714 if (block2)
98fa08a5 1715 lightrec_recompiler_remove(state->rec, block2);
fd58fa32 1716 }
98fa08a5 1717
fd58fa32
PC		 1718 /* We know from now on that block2 (if present) isn't going to
1719 * be compiled. We can override the LUT entry with our new
1720 * block's entry point. */
1721 offset = lut_offset(block->pc) + target->offset;
02487de7 1722 lut_write(state, offset, jit_address(target->label));
98fa08a5 1723
fd58fa32 1724 if (block2) {
a59e5536 1725 pr_debug("Reap block 0x%08x as it's covered by block "
1726 "0x%08x\n", block2->pc, block->pc);
1727
98fa08a5 1728 /* Finally, reap the block. */
ba3814c1
PC		 1729 if (!ENABLE_THREADED_COMPILER) {
1730 lightrec_unregister_block(state->block_cache, block2);
1731 lightrec_free_block(state, block2);
1732 } else if (!(old_flags & BLOCK_IS_DEAD)) {
a59e5536 1733 lightrec_reaper_add(state->reaper,
1734 lightrec_reap_block,
1735 block2);
a59e5536 1736 }
1737 }
1738 }
1739
d16005f8 1740 if (ENABLE_DISASSEMBLER) {
98fa08a5 1741 pr_debug("Compiling block at PC: 0x%08x\n", block->pc);
d16005f8
PC		 1742 jit_disassemble();
1743 }
1744
1745 jit_clear_state();
1746
d16005f8 1747 if (fully_tagged)
ba3814c1
PC		 1748 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
1749
1750 if (fully_tagged && !(old_flags & BLOCK_NO_OPCODE_LIST)) {
f5ee77ca 1751 pr_debug("Block "PC_FMT" is fully tagged"
d16005f8 1752 " - free opcode list\n", block->pc);
ba3814c1
PC		 1753
1754 if (ENABLE_THREADED_COMPILER) {
1755 lightrec_reaper_add(state->reaper,
1756 lightrec_reap_opcode_list,
1757 block->opcode_list);
1758 } else {
1759 lightrec_free_opcode_list(state, block->opcode_list);
1760 }
d16005f8
PC		 1761 }
1762
a59e5536 1763 if (oldjit) {
1764 pr_debug("Block 0x%08x recompiled, reaping old jit context.\n",
1765 block->pc);
1766
02487de7 1767 if (ENABLE_THREADED_COMPILER) {
a59e5536 1768 lightrec_reaper_add(state->reaper,
1769 lightrec_reap_jit, oldjit);
02487de7
PC		 1770 lightrec_reaper_add(state->reaper,
1771 lightrec_reap_function, old_fn);
1772 } else {
a59e5536 1773 _jit_destroy_state(oldjit);
02487de7
PC		 1774 lightrec_free_function(state, old_fn);
1775 }
ba3814c1
PC		 1776
1777 lightrec_unregister(MEM_FOR_CODE, old_code_size);
a59e5536 1778 }
1779
cb72ea13
PC		 1780 pr_debug("Blocks compiled: %u\n", ++state->nb_compile);
1781
d16005f8
PC		 1782 return 0;
1783}
1784
98fa08a5
PC		 1785static void lightrec_print_info(struct lightrec_state *state)
1786{
1787 if ((state->current_cycle & ~0xfffffff) != state->old_cycle_counter) {
1788 pr_info("Lightrec RAM usage: IR %u KiB, CODE %u KiB, "
1789 "MIPS %u KiB, TOTAL %u KiB, avg. IPI %f\n",
1790 lightrec_get_mem_usage(MEM_FOR_IR) / 1024,
1791 lightrec_get_mem_usage(MEM_FOR_CODE) / 1024,
1792 lightrec_get_mem_usage(MEM_FOR_MIPS_CODE) / 1024,
1793 lightrec_get_total_mem_usage() / 1024,
1794 lightrec_get_average_ipi());
1795 state->old_cycle_counter = state->current_cycle & ~0xfffffff;
1796 }
1797}
1798
d16005f8
PC		 1799u32 lightrec_execute(struct lightrec_state *state, u32 pc, u32 target_cycle)
1800{
9259d748 1801 s32 (*func)(struct lightrec_state *, u32, void *, s32) = (void *)state->dispatcher->function;
d16005f8
PC		 1802 void *block_trace;
1803 s32 cycles_delta;
1804
1805 state->exit_flags = LIGHTREC_EXIT_NORMAL;
1806
1807 /* Handle the cycle counter overflowing */
1808 if (unlikely(target_cycle < state->current_cycle))
1809 target_cycle = UINT_MAX;
1810
1811 state->target_cycle = target_cycle;
0e720fb1 1812 state->curr_pc = pc;
d16005f8
PC		 1813
1814 block_trace = get_next_block_func(state, pc);
1815 if (block_trace) {
1816 cycles_delta = state->target_cycle - state->current_cycle;
1817
0e720fb1 1818 cycles_delta = (*func)(state, state->curr_pc,
9259d748 1819 block_trace, cycles_delta);
d16005f8
PC		 1820
1821 state->current_cycle = state->target_cycle - cycles_delta;
1822 }
1823
a59e5536 1824 if (ENABLE_THREADED_COMPILER)
1825 lightrec_reaper_reap(state->reaper);
1826
98fa08a5
PC		 1827 if (LOG_LEVEL >= INFO_L)
1828 lightrec_print_info(state);
1829
0e720fb1 1830 return state->curr_pc;
d16005f8
PC		 1831}
1832
ba3814c1
PC		 1833u32 lightrec_run_interpreter(struct lightrec_state *state, u32 pc,
1834 u32 target_cycle)
d16005f8 1835{
ba3814c1 1836 struct block *block;
d16005f8
PC		 1837
1838 state->exit_flags = LIGHTREC_EXIT_NORMAL;
ba3814c1
PC		 1839 state->target_cycle = target_cycle;
1840
1841 do {
1842 block = lightrec_get_block(state, pc);
1843 if (!block)
1844 break;
1845
1846 pc = lightrec_emulate_block(state, block, pc);
d16005f8 1847
ba3814c1
PC		 1848 if (ENABLE_THREADED_COMPILER)
1849 lightrec_reaper_reap(state->reaper);
1850 } while (state->current_cycle < state->target_cycle);
98fa08a5
PC		 1851
1852 if (LOG_LEVEL >= INFO_L)
1853 lightrec_print_info(state);
1854
1855 return pc;
d16005f8
PC		 1856}
1857
98fa08a5 1858void lightrec_free_block(struct lightrec_state *state, struct block *block)
d16005f8 1859{
ba3814c1
PC		 1860 u8 old_flags;
1861
d16005f8 1862 lightrec_unregister(MEM_FOR_MIPS_CODE, block->nb_ops * sizeof(u32));
ba3814c1
PC		 1863 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
1864
1865 if (!(old_flags & BLOCK_NO_OPCODE_LIST))
1866 lightrec_free_opcode_list(state, block->opcode_list);
d16005f8
PC		 1867 if (block->_jit)
1868 _jit_destroy_state(block->_jit);
d8b04acd
PC		 1869 if (block->function) {
1870 lightrec_free_function(state, block->function);
1871 lightrec_unregister(MEM_FOR_CODE, block->code_size);
1872 }
98fa08a5
PC		 1873 lightrec_free(state, MEM_FOR_IR, sizeof(*block), block);
1874}
1875
1876struct lightrec_cstate * lightrec_create_cstate(struct lightrec_state *state)
1877{
1878 struct lightrec_cstate *cstate;
1879
1880 cstate = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*cstate));
1881 if (!cstate)
1882 return NULL;
1883
1884 cstate->reg_cache = lightrec_regcache_init(state);
1885 if (!cstate->reg_cache) {
1886 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*cstate), cstate);
1887 return NULL;
1888 }
1889
1890 cstate->state = state;
1891
1892 return cstate;
1893}
1894
1895void lightrec_free_cstate(struct lightrec_cstate *cstate)
1896{
1897 lightrec_free_regcache(cstate->reg_cache);
1898 lightrec_free(cstate->state, MEM_FOR_LIGHTREC, sizeof(*cstate), cstate);
d16005f8
PC		 1899}
1900
1901struct lightrec_state * lightrec_init(char *argv0,
1902 const struct lightrec_mem_map *map,
1903 size_t nb,
1904 const struct lightrec_ops *ops)
1905{
d8b04acd 1906 const struct lightrec_mem_map *codebuf_map = &map[PSX_MAP_CODE_BUFFER];
d16005f8 1907 struct lightrec_state *state;
02487de7
PC		 1908 uintptr_t addr;
1909 void *tlsf = NULL;
1910 bool with_32bit_lut = false;
1911 size_t lut_size;
d16005f8
PC		 1912
1913 /* Sanity-check ops */
98fa08a5 1914 if (!ops || !ops->cop2_op || !ops->enable_ram) {
d16005f8
PC		 1915 pr_err("Missing callbacks in lightrec_ops structure\n");
1916 return NULL;
1917 }
1918
fdf33147
PC		 1919 if (ops->cop2_notify)
1920 pr_debug("Optional cop2_notify callback in lightrec_ops\n");
1921 else
1922 pr_debug("No optional cop2_notify callback in lightrec_ops\n");
1923
d8b04acd
PC		 1924 if (ENABLE_CODE_BUFFER && nb > PSX_MAP_CODE_BUFFER
1925 && codebuf_map->address) {
02487de7
PC		 1926 tlsf = tlsf_create_with_pool(codebuf_map->address,
1927 codebuf_map->length);
1928 if (!tlsf) {
1929 pr_err("Unable to initialize code buffer\n");
1930 return NULL;
1931 }
1932
1933 if (__WORDSIZE == 64) {
1934 addr = (uintptr_t) codebuf_map->address + codebuf_map->length - 1;
1935 with_32bit_lut = addr == (u32) addr;
1936 }
1937 }
1938
1939 if (with_32bit_lut)
1940 lut_size = CODE_LUT_SIZE * 4;
1941 else
1942 lut_size = CODE_LUT_SIZE * sizeof(void *);
1943
fb67ea33 1944 init_jit_with_debug(argv0, stdout);
d16005f8 1945
02487de7 1946 state = calloc(1, sizeof(*state) + lut_size);
d16005f8
PC		 1947 if (!state)
1948 goto err_finish_jit;
1949
02487de7
PC		 1950 lightrec_register(MEM_FOR_LIGHTREC, sizeof(*state) + lut_size);
1951
1952 state->tlsf = tlsf;
1953 state->with_32bit_lut = with_32bit_lut;
cb72ea13 1954 state->in_delay_slot_n = 0xff;
684432ad 1955 state->cycles_per_op = 2;
d16005f8 1956
d16005f8
PC		 1957 state->block_cache = lightrec_blockcache_init(state);
1958 if (!state->block_cache)
d8b04acd 1959 goto err_free_state;
d16005f8 1960
d16005f8
PC		 1961 if (ENABLE_THREADED_COMPILER) {
1962 state->rec = lightrec_recompiler_init(state);
1963 if (!state->rec)
98fa08a5 1964 goto err_free_block_cache;
a59e5536 1965
1966 state->reaper = lightrec_reaper_init(state);
1967 if (!state->reaper)
1968 goto err_free_recompiler;
98fa08a5
PC		 1969 } else {
1970 state->cstate = lightrec_create_cstate(state);
1971 if (!state->cstate)
1972 goto err_free_block_cache;
d16005f8
PC		 1973 }
1974
1975 state->nb_maps = nb;
1976 state->maps = map;
1977
1978 memcpy(&state->ops, ops, sizeof(*ops));
1979
1980 state->dispatcher = generate_dispatcher(state);
1981 if (!state->dispatcher)
a59e5536 1982 goto err_free_reaper;
d16005f8 1983
98fa08a5
PC		 1984 state->c_wrapper_block = generate_wrapper(state);
1985 if (!state->c_wrapper_block)
d16005f8
PC		 1986 goto err_free_dispatcher;
1987
98fa08a5
PC		 1988 state->c_wrappers[C_WRAPPER_RW] = lightrec_rw_cb;
1989 state->c_wrappers[C_WRAPPER_RW_GENERIC] = lightrec_rw_generic_cb;
fdf33147 1990 state->c_wrappers[C_WRAPPER_MFC] = lightrec_mfc_cb;
98fa08a5 1991 state->c_wrappers[C_WRAPPER_MTC] = lightrec_mtc_cb;
22eee2ac 1992 state->c_wrappers[C_WRAPPER_CP] = lightrec_cp_cb;
d16005f8
PC		 1993
1994 map = &state->maps[PSX_MAP_BIOS];
1995 state->offset_bios = (uintptr_t)map->address - map->pc;
1996
1997 map = &state->maps[PSX_MAP_SCRATCH_PAD];
1998 state->offset_scratch = (uintptr_t)map->address - map->pc;
1999
ba3814c1
PC		 2000 map = &state->maps[PSX_MAP_HW_REGISTERS];
2001 state->offset_io = (uintptr_t)map->address - map->pc;
2002
d16005f8
PC		 2003 map = &state->maps[PSX_MAP_KERNEL_USER_RAM];
2004 state->offset_ram = (uintptr_t)map->address - map->pc;
2005
2006 if (state->maps[PSX_MAP_MIRROR1].address == map->address + 0x200000 &&
2007 state->maps[PSX_MAP_MIRROR2].address == map->address + 0x400000 &&
2008 state->maps[PSX_MAP_MIRROR3].address == map->address + 0x600000)
2009 state->mirrors_mapped = true;
2010
98fa08a5
PC		 2011 if (state->offset_bios == 0 &&
2012 state->offset_scratch == 0 &&
2013 state->offset_ram == 0 &&
ba3814c1 2014 state->offset_io == 0 &&
98fa08a5
PC		 2015 state->mirrors_mapped) {
2016 pr_info("Memory map is perfect. Emitted code will be best.\n");
2017 } else {
2018 pr_info("Memory map is sub-par. Emitted code will be slow.\n");
2019 }
2020
02487de7
PC		 2021 if (state->with_32bit_lut)
2022 pr_info("Using 32-bit LUT\n");
2023
d16005f8
PC		 2024 return state;
2025
d16005f8 2026err_free_dispatcher:
98fa08a5 2027 lightrec_free_block(state, state->dispatcher);
a59e5536 2028err_free_reaper:
2029 if (ENABLE_THREADED_COMPILER)
2030 lightrec_reaper_destroy(state->reaper);
d16005f8
PC		 2031err_free_recompiler:
2032 if (ENABLE_THREADED_COMPILER)
2033 lightrec_free_recompiler(state->rec);
98fa08a5
PC		 2034 else
2035 lightrec_free_cstate(state->cstate);
d16005f8
PC		 2036err_free_block_cache:
2037 lightrec_free_block_cache(state->block_cache);
d16005f8 2038err_free_state:
d16005f8 2039 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
02487de7 2040 lut_elm_size(state) * CODE_LUT_SIZE);
d16005f8
PC		 2041 free(state);
2042err_finish_jit:
2043 finish_jit();
02487de7
PC		 2044 if (ENABLE_CODE_BUFFER && tlsf)
2045 tlsf_destroy(tlsf);
d16005f8
PC		 2046 return NULL;
2047}
2048
2049void lightrec_destroy(struct lightrec_state *state)
2050{
98fa08a5
PC		 2051 /* Force a print info on destroy*/
2052 state->current_cycle = ~state->current_cycle;
2053 lightrec_print_info(state);
2054
03535202
PC		 2055 lightrec_free_block_cache(state->block_cache);
2056 lightrec_free_block(state, state->dispatcher);
2057 lightrec_free_block(state, state->c_wrapper_block);
2058
a59e5536 2059 if (ENABLE_THREADED_COMPILER) {
d16005f8 2060 lightrec_free_recompiler(state->rec);
a59e5536 2061 lightrec_reaper_destroy(state->reaper);
98fa08a5
PC		 2062 } else {
2063 lightrec_free_cstate(state->cstate);
a59e5536 2064 }
d16005f8 2065
d16005f8 2066 finish_jit();
02487de7
PC		 2067 if (ENABLE_CODE_BUFFER && state->tlsf)
2068 tlsf_destroy(state->tlsf);
d16005f8 2069
d16005f8 2070 lightrec_unregister(MEM_FOR_LIGHTREC, sizeof(*state) +
02487de7 2071 lut_elm_size(state) * CODE_LUT_SIZE);
d16005f8
PC		 2072 free(state);
2073}
2074
2075void lightrec_invalidate(struct lightrec_state *state, u32 addr, u32 len)
2076{
2077 u32 kaddr = kunseg(addr & ~0x3);
d8b04acd 2078 enum psx_map idx = lightrec_get_map_idx(state, kaddr);
d16005f8 2079
d8b04acd
PC		 2080 switch (idx) {
2081 case PSX_MAP_MIRROR1:
2082 case PSX_MAP_MIRROR2:
2083 case PSX_MAP_MIRROR3:
d16005f8 2084 /* Handle mirrors */
d8b04acd
PC		 2085 kaddr &= RAM_SIZE - 1;
2086 fallthrough;
2087 case PSX_MAP_KERNEL_USER_RAM:
2088 break;
2089 default:
2090 return;
d16005f8 2091 }
d8b04acd
PC		 2092
2093 memset(lut_address(state, lut_offset(kaddr)), 0,
2094 ((len + 3) / 4) * lut_elm_size(state));
d16005f8
PC		 2095}
2096
2097void lightrec_invalidate_all(struct lightrec_state *state)
2098{
02487de7 2099 memset(state->code_lut, 0, lut_elm_size(state) * CODE_LUT_SIZE);
d16005f8
PC		 2100}
2101
684432ad 2102void lightrec_set_unsafe_opt_flags(struct lightrec_state *state, u32 flags)
d16005f8 2103{
684432ad 2104 if ((flags ^ state->opt_flags) & LIGHTREC_OPT_INV_DMA_ONLY)
d16005f8
PC		 2105 lightrec_invalidate_all(state);
2106
684432ad 2107 state->opt_flags = flags;
d16005f8
PC		 2108}
2109
2110void lightrec_set_exit_flags(struct lightrec_state *state, u32 flags)
2111{
2112 if (flags != LIGHTREC_EXIT_NORMAL) {
2113 state->exit_flags |= flags;
2114 state->target_cycle = state->current_cycle;
2115 }
2116}
2117
2118u32 lightrec_exit_flags(struct lightrec_state *state)
2119{
2120 return state->exit_flags;
2121}
2122
d16005f8
PC		 2123u32 lightrec_current_cycle_count(const struct lightrec_state *state)
2124{
2125 return state->current_cycle;
2126}
2127
2128void lightrec_reset_cycle_count(struct lightrec_state *state, u32 cycles)
2129{
2130 state->current_cycle = cycles;
2131
2132 if (state->target_cycle < cycles)
2133 state->target_cycle = cycles;
2134}
2135
2136void lightrec_set_target_cycle_count(struct lightrec_state *state, u32 cycles)
2137{
2138 if (state->exit_flags == LIGHTREC_EXIT_NORMAL) {
2139 if (cycles < state->current_cycle)
2140 cycles = state->current_cycle;
2141
2142 state->target_cycle = cycles;
2143 }
2144}
98fa08a5
PC		 2145
2146struct lightrec_registers * lightrec_get_registers(struct lightrec_state *state)
2147{
2148 return &state->regs;
2149}
684432ad
PC		 2150
2151void lightrec_set_cycles_per_opcode(struct lightrec_state *state, u32 cycles)
2152{
2153 state->cycles_per_op = cycles;
2154}
ARM optimized PCSX fork