libretro: direct fb access requires duping support
[pcsx_rearmed.git] / deps / lightrec / recompiler.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC
4 */
5
d8b04acd 6#include "blockcache.h"
d16005f8
PC
7#include "debug.h"
8#include "interpreter.h"
9#include "lightrec-private.h"
10#include "memmanager.h"
d8b04acd 11#include "reaper.h"
a59e5536 12#include "slist.h"
d16005f8
PC
13
14#include <errno.h>
15#include <stdatomic.h>
16#include <stdbool.h>
17#include <stdlib.h>
18#include <pthread.h>
98fa08a5
PC
19#ifdef __linux__
20#include <unistd.h>
21#endif
d16005f8
PC
22
23struct block_rec {
24 struct block *block;
a59e5536 25 struct slist_elm slist;
98fa08a5
PC
26 bool compiling;
27};
28
29struct recompiler_thd {
30 struct lightrec_cstate *cstate;
31 unsigned int tid;
32 pthread_t thd;
d16005f8
PC
33};
34
35struct recompiler {
36 struct lightrec_state *state;
d16005f8 37 pthread_cond_t cond;
98fa08a5 38 pthread_cond_t cond2;
d16005f8 39 pthread_mutex_t mutex;
d8b04acd 40 bool stop, must_flush;
a59e5536 41 struct slist_elm slist;
98fa08a5 42
d8b04acd
PC
43 pthread_mutex_t alloc_mutex;
44
98fa08a5
PC
45 unsigned int nb_recs;
46 struct recompiler_thd thds[];
d16005f8
PC
47};
48
98fa08a5
PC
49static unsigned int get_processors_count(void)
50{
11357fef 51 unsigned int nb = 1;
98fa08a5
PC
52
53#if defined(PTW32_VERSION)
54 nb = pthread_num_processors_np();
55#elif defined(__APPLE__) || defined(__FreeBSD__)
56 int count;
57 size_t size = sizeof(count);
58
59 nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
11357fef 60#elif defined(_SC_NPROCESSORS_ONLN)
98fa08a5
PC
61 nb = sysconf(_SC_NPROCESSORS_ONLN);
62#endif
63
64 return nb < 1 ? 1 : nb;
65}
66
67static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
68{
69 struct block_rec *block_rec;
70 struct slist_elm *elm;
71
72 for (elm = slist_first(head); elm; elm = elm->next) {
73 block_rec = container_of(elm, struct block_rec, slist);
74
75 if (!block_rec->compiling)
76 return elm;
77 }
78
79 return NULL;
80}
81
d8b04acd
PC
82static bool lightrec_cancel_block_rec(struct recompiler *rec,
83 struct block_rec *block_rec)
84{
85 if (block_rec->compiling) {
86 /* Block is being recompiled - wait for
87 * completion */
88 pthread_cond_wait(&rec->cond2, &rec->mutex);
89
90 /* We can't guarantee the signal was for us.
91 * Since block_rec may have been removed while
92 * we were waiting on the condition, we cannot
93 * check block_rec->compiling again. The best
94 * thing is just to restart the function. */
95 return false;
96 }
97
98 /* Block is not yet being processed - remove it from the list */
99 slist_remove(&rec->slist, &block_rec->slist);
100 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
101 sizeof(*block_rec), block_rec);
102
103 return true;
104}
105
106static void lightrec_cancel_list(struct recompiler *rec)
107{
108 struct block_rec *block_rec;
ba3814c1 109 struct slist_elm *elm, *head = &rec->slist;
d8b04acd 110
ba3814c1
PC
111 for (elm = slist_first(head); elm; elm = slist_first(head)) {
112 block_rec = container_of(elm, struct block_rec, slist);
d8b04acd
PC
113 lightrec_cancel_block_rec(rec, block_rec);
114 }
d8b04acd
PC
115}
116
117static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d)
118{
119 struct recompiler *rec = d;
120
ba3814c1
PC
121 lightrec_remove_outdated_blocks(state->block_cache, NULL);
122 rec->must_flush = false;
d8b04acd
PC
123}
124
98fa08a5
PC
125static void lightrec_compile_list(struct recompiler *rec,
126 struct recompiler_thd *thd)
d16005f8 127{
a59e5536 128 struct block_rec *block_rec;
129 struct slist_elm *next;
d16005f8
PC
130 struct block *block;
131 int ret;
132
98fa08a5 133 while (!!(next = lightrec_get_first_elm(&rec->slist))) {
a59e5536 134 block_rec = container_of(next, struct block_rec, slist);
98fa08a5 135 block_rec->compiling = true;
a59e5536 136 block = block_rec->block;
d16005f8
PC
137
138 pthread_mutex_unlock(&rec->mutex);
139
ba3814c1 140 if (likely(!block_has_flag(block, BLOCK_IS_DEAD))) {
98fa08a5 141 ret = lightrec_compile_block(thd->cstate, block);
d8b04acd
PC
142 if (ret == -ENOMEM) {
143 /* Code buffer is full. Request the reaper to
144 * flush it. */
145
146 pthread_mutex_lock(&rec->mutex);
ba3814c1
PC
147 block_rec->compiling = false;
148 pthread_cond_broadcast(&rec->cond2);
149
d8b04acd 150 if (!rec->must_flush) {
ba3814c1
PC
151 rec->must_flush = true;
152 lightrec_cancel_list(rec);
153
d8b04acd
PC
154 lightrec_reaper_add(rec->state->reaper,
155 lightrec_flush_code_buffer,
156 rec);
d8b04acd
PC
157 }
158 return;
159 }
160
98fa08a5 161 if (ret) {
f5ee77ca 162 pr_err("Unable to compile block at "PC_FMT": %d\n",
98fa08a5
PC
163 block->pc, ret);
164 }
d16005f8
PC
165 }
166
167 pthread_mutex_lock(&rec->mutex);
168
a59e5536 169 slist_remove(&rec->slist, next);
d16005f8 170 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536 171 sizeof(*block_rec), block_rec);
ba3814c1 172 pthread_cond_broadcast(&rec->cond2);
d16005f8 173 }
d16005f8
PC
174}
175
176static void * lightrec_recompiler_thd(void *d)
177{
98fa08a5
PC
178 struct recompiler_thd *thd = d;
179 struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8
PC
180
181 pthread_mutex_lock(&rec->mutex);
182
ea17432f 183 while (!rec->stop) {
d16005f8
PC
184 do {
185 pthread_cond_wait(&rec->cond, &rec->mutex);
186
a59e5536 187 if (rec->stop)
188 goto out_unlock;
d16005f8 189
a59e5536 190 } while (slist_empty(&rec->slist));
d16005f8 191
98fa08a5 192 lightrec_compile_list(rec, thd);
ea17432f 193 }
a59e5536 194
195out_unlock:
196 pthread_mutex_unlock(&rec->mutex);
197 return NULL;
d16005f8
PC
198}
199
200struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
201{
202 struct recompiler *rec;
98fa08a5 203 unsigned int i, nb_recs, nb_cpus;
d16005f8
PC
204 int ret;
205
98fa08a5
PC
206 nb_cpus = get_processors_count();
207 nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
208
209 rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
210 + nb_recs * sizeof(*rec->thds));
d16005f8
PC
211 if (!rec) {
212 pr_err("Cannot create recompiler: Out of memory\n");
213 return NULL;
214 }
215
98fa08a5
PC
216 for (i = 0; i < nb_recs; i++) {
217 rec->thds[i].tid = i;
218 rec->thds[i].cstate = NULL;
219 }
220
221 for (i = 0; i < nb_recs; i++) {
222 rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e 223 if (!rec->thds[i].cstate) {
98fa08a5
PC
224 pr_err("Cannot create recompiler: Out of memory\n");
225 goto err_free_cstates;
226 }
227 }
228
d16005f8
PC
229 rec->state = state;
230 rec->stop = false;
d8b04acd 231 rec->must_flush = false;
98fa08a5 232 rec->nb_recs = nb_recs;
a59e5536 233 slist_init(&rec->slist);
d16005f8
PC
234
235 ret = pthread_cond_init(&rec->cond, NULL);
236 if (ret) {
237 pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5 238 goto err_free_cstates;
d16005f8
PC
239 }
240
98fa08a5 241 ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8 242 if (ret) {
98fa08a5 243 pr_err("Cannot init cond variable: %d\n", ret);
d16005f8
PC
244 goto err_cnd_destroy;
245 }
246
d8b04acd
PC
247 ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
248 if (ret) {
249 pr_err("Cannot init alloc mutex variable: %d\n", ret);
250 goto err_cnd2_destroy;
251 }
252
98fa08a5 253 ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8 254 if (ret) {
98fa08a5 255 pr_err("Cannot init mutex variable: %d\n", ret);
d8b04acd 256 goto err_alloc_mtx_destroy;
d16005f8
PC
257 }
258
98fa08a5
PC
259 for (i = 0; i < nb_recs; i++) {
260 ret = pthread_create(&rec->thds[i].thd, NULL,
261 lightrec_recompiler_thd, &rec->thds[i]);
262 if (ret) {
263 pr_err("Cannot create recompiler thread: %d\n", ret);
264 /* TODO: Handle cleanup properly */
265 goto err_mtx_destroy;
266 }
267 }
268
269 pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
270
d16005f8
PC
271 return rec;
272
273err_mtx_destroy:
274 pthread_mutex_destroy(&rec->mutex);
d8b04acd
PC
275err_alloc_mtx_destroy:
276 pthread_mutex_destroy(&rec->alloc_mutex);
98fa08a5
PC
277err_cnd2_destroy:
278 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
279err_cnd_destroy:
280 pthread_cond_destroy(&rec->cond);
98fa08a5
PC
281err_free_cstates:
282 for (i = 0; i < nb_recs; i++) {
283 if (rec->thds[i].cstate)
284 lightrec_free_cstate(rec->thds[i].cstate);
285 }
d16005f8
PC
286 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
287 return NULL;
288}
289
290void lightrec_free_recompiler(struct recompiler *rec)
291{
98fa08a5
PC
292 unsigned int i;
293
d16005f8
PC
294 rec->stop = true;
295
296 /* Stop the thread */
297 pthread_mutex_lock(&rec->mutex);
98fa08a5 298 pthread_cond_broadcast(&rec->cond);
d8b04acd 299 lightrec_cancel_list(rec);
d16005f8 300 pthread_mutex_unlock(&rec->mutex);
98fa08a5
PC
301
302 for (i = 0; i < rec->nb_recs; i++)
303 pthread_join(rec->thds[i].thd, NULL);
304
305 for (i = 0; i < rec->nb_recs; i++)
306 lightrec_free_cstate(rec->thds[i].cstate);
d16005f8
PC
307
308 pthread_mutex_destroy(&rec->mutex);
d8b04acd 309 pthread_mutex_destroy(&rec->alloc_mutex);
d16005f8 310 pthread_cond_destroy(&rec->cond);
98fa08a5 311 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
312 lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
313}
314
315int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
316{
a59e5536 317 struct slist_elm *elm, *prev;
318 struct block_rec *block_rec;
319 int ret = 0;
d16005f8
PC
320
321 pthread_mutex_lock(&rec->mutex);
322
d8b04acd
PC
323 /* If the recompiler must flush the code cache, we can't add the new
324 * job. It will be re-added next time the block's address is jumped to
325 * again. */
326 if (rec->must_flush)
327 goto out_unlock;
328
a59e5536 329 /* If the block is marked as dead, don't compile it, it will be removed
330 * as soon as it's safe. */
ba3814c1 331 if (block_has_flag(block, BLOCK_IS_DEAD))
a59e5536 332 goto out_unlock;
333
334 for (elm = slist_first(&rec->slist), prev = NULL; elm;
335 prev = elm, elm = elm->next) {
336 block_rec = container_of(elm, struct block_rec, slist);
337
d16005f8
PC
338 if (block_rec->block == block) {
339 /* The block to compile is already in the queue - bump
a59e5536 340 * it to the top of the list, unless the block is being
341 * recompiled. */
98fa08a5 342 if (prev && !block_rec->compiling &&
ba3814c1 343 !block_has_flag(block, BLOCK_SHOULD_RECOMPILE)) {
a59e5536 344 slist_remove_next(prev);
345 slist_append(&rec->slist, elm);
d16005f8
PC
346 }
347
a59e5536 348 goto out_unlock;
d16005f8
PC
349 }
350 }
351
352 /* By the time this function was called, the block has been recompiled
353 * and ins't in the wait list anymore. Just return here. */
ba3814c1 354 if (block->function && !block_has_flag(block, BLOCK_SHOULD_RECOMPILE))
a59e5536 355 goto out_unlock;
d16005f8
PC
356
357 block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
358 sizeof(*block_rec));
359 if (!block_rec) {
a59e5536 360 ret = -ENOMEM;
361 goto out_unlock;
d16005f8
PC
362 }
363
f5ee77ca 364 pr_debug("Adding block "PC_FMT" to recompiler\n", block->pc);
d16005f8
PC
365
366 block_rec->block = block;
98fa08a5 367 block_rec->compiling = false;
a59e5536 368
369 elm = &rec->slist;
370
371 /* If the block is being recompiled, push it to the end of the queue;
372 * otherwise push it to the front of the queue. */
ba3814c1 373 if (block_has_flag(block, BLOCK_SHOULD_RECOMPILE))
a59e5536 374 for (; elm->next; elm = elm->next);
375
376 slist_append(elm, &block_rec->slist);
d16005f8
PC
377
378 /* Signal the thread */
379 pthread_cond_signal(&rec->cond);
d16005f8 380
a59e5536 381out_unlock:
382 pthread_mutex_unlock(&rec->mutex);
98fa08a5 383
a59e5536 384 return ret;
d16005f8
PC
385}
386
387void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
388{
389 struct block_rec *block_rec;
a59e5536 390 struct slist_elm *elm;
d16005f8
PC
391
392 pthread_mutex_lock(&rec->mutex);
393
98fa08a5
PC
394 while (true) {
395 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
396 block_rec = container_of(elm, struct block_rec, slist);
a59e5536 397
d8b04acd
PC
398 if (block_rec->block == block) {
399 if (lightrec_cancel_block_rec(rec, block_rec))
400 goto out_unlock;
98fa08a5 401
98fa08a5 402 break;
d16005f8 403 }
98fa08a5 404 }
d16005f8 405
98fa08a5 406 if (!elm)
d16005f8 407 break;
d16005f8
PC
408 }
409
98fa08a5 410out_unlock:
d16005f8
PC
411 pthread_mutex_unlock(&rec->mutex);
412}
413
98fa08a5
PC
414void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
415 struct block *block, u32 *pc)
d16005f8 416{
ba3814c1 417 u8 old_flags;
d16005f8 418
98fa08a5
PC
419 /* There's no point in running the first pass if the block will never
420 * be compiled. Let the main loop run the interpreter instead. */
ba3814c1 421 if (block_has_flag(block, BLOCK_NEVER_COMPILE))
98fa08a5
PC
422 return NULL;
423
ba3814c1
PC
424 /* The block is marked as dead, and will be removed the next time the
425 * reaper is run. In the meantime, the old function can still be
426 * executed. */
427 if (block_has_flag(block, BLOCK_IS_DEAD))
428 return block->function;
429
98fa08a5
PC
430 /* If the block is already fully tagged, there is no point in running
431 * the first pass. Request a recompilation of the block, and maybe the
432 * interpreter will run the block in the meantime. */
ba3814c1 433 if (block_has_flag(block, BLOCK_FULLY_TAGGED))
98fa08a5
PC
434 lightrec_recompiler_add(state->rec, block);
435
d16005f8 436 if (likely(block->function)) {
ba3814c1
PC
437 if (block_has_flag(block, BLOCK_FULLY_TAGGED)) {
438 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8 439
ba3814c1 440 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
f5ee77ca 441 pr_debug("Block "PC_FMT" is fully tagged"
d16005f8
PC
442 " - free opcode list\n", block->pc);
443
444 /* The block was already compiled but the opcode list
445 * didn't get freed yet - do it now */
ba3814c1 446 lightrec_free_opcode_list(state, block->opcode_list);
d16005f8
PC
447 }
448 }
449
450 return block->function;
451 }
452
453 /* Mark the opcode list as freed, so that the threaded compiler won't
454 * free it while we're using it in the interpreter. */
ba3814c1 455 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8
PC
456
457 /* Block wasn't compiled yet - run the interpreter */
98fa08a5 458 *pc = lightrec_emulate_block(state, block, *pc);
d16005f8 459
ba3814c1
PC
460 if (!(old_flags & BLOCK_NO_OPCODE_LIST))
461 block_clear_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8
PC
462
463 /* The block got compiled while the interpreter was running.
464 * We can free the opcode list now. */
ba3814c1
PC
465 if (block->function && block_has_flag(block, BLOCK_FULLY_TAGGED)) {
466 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8 467
ba3814c1 468 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
f5ee77ca 469 pr_debug("Block "PC_FMT" is fully tagged"
ba3814c1
PC
470 " - free opcode list\n", block->pc);
471
472 lightrec_free_opcode_list(state, block->opcode_list);
473 }
d16005f8
PC
474 }
475
476 return NULL;
477}
d8b04acd
PC
478
479void lightrec_code_alloc_lock(struct lightrec_state *state)
480{
481 pthread_mutex_lock(&state->rec->alloc_mutex);
482}
483
484void lightrec_code_alloc_unlock(struct lightrec_state *state)
485{
486 pthread_mutex_unlock(&state->rec->alloc_mutex);
487}