git subrepo pull --force deps/lightrec
[pcsx_rearmed.git] / deps / lightrec / recompiler.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC
4 */
5
d8b04acd 6#include "blockcache.h"
d16005f8
PC
7#include "debug.h"
8#include "interpreter.h"
9#include "lightrec-private.h"
10#include "memmanager.h"
d8b04acd 11#include "reaper.h"
a59e5536 12#include "slist.h"
d16005f8
PC
13
14#include <errno.h>
15#include <stdatomic.h>
16#include <stdbool.h>
17#include <stdlib.h>
18#include <pthread.h>
98fa08a5
PC
19#ifdef __linux__
20#include <unistd.h>
21#endif
d16005f8
PC
22
23struct block_rec {
24 struct block *block;
a59e5536 25 struct slist_elm slist;
a5a6f7b8 26 unsigned int requests;
98fa08a5
PC
27 bool compiling;
28};
29
30struct recompiler_thd {
31 struct lightrec_cstate *cstate;
32 unsigned int tid;
33 pthread_t thd;
d16005f8
PC
34};
35
36struct recompiler {
37 struct lightrec_state *state;
d16005f8 38 pthread_cond_t cond;
98fa08a5 39 pthread_cond_t cond2;
d16005f8 40 pthread_mutex_t mutex;
d8b04acd 41 bool stop, must_flush;
a59e5536 42 struct slist_elm slist;
98fa08a5 43
d8b04acd
PC
44 pthread_mutex_t alloc_mutex;
45
98fa08a5
PC
46 unsigned int nb_recs;
47 struct recompiler_thd thds[];
d16005f8
PC
48};
49
98fa08a5
PC
50static unsigned int get_processors_count(void)
51{
11357fef 52 unsigned int nb = 1;
98fa08a5
PC
53
54#if defined(PTW32_VERSION)
55 nb = pthread_num_processors_np();
56#elif defined(__APPLE__) || defined(__FreeBSD__)
57 int count;
58 size_t size = sizeof(count);
59
60 nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
11357fef 61#elif defined(_SC_NPROCESSORS_ONLN)
98fa08a5
PC
62 nb = sysconf(_SC_NPROCESSORS_ONLN);
63#endif
64
65 return nb < 1 ? 1 : nb;
66}
67
a5a6f7b8 68static struct block_rec * lightrec_get_best_elm(struct slist_elm *head)
98fa08a5 69{
a5a6f7b8 70 struct block_rec *block_rec, *best = NULL;
98fa08a5
PC
71 struct slist_elm *elm;
72
73 for (elm = slist_first(head); elm; elm = elm->next) {
74 block_rec = container_of(elm, struct block_rec, slist);
75
a5a6f7b8
PC
76 if (!block_rec->compiling
77 && (!best || block_rec->requests > best->requests))
78 best = block_rec;
98fa08a5
PC
79 }
80
a5a6f7b8 81 return best;
98fa08a5
PC
82}
83
d8b04acd
PC
84static bool lightrec_cancel_block_rec(struct recompiler *rec,
85 struct block_rec *block_rec)
86{
87 if (block_rec->compiling) {
88 /* Block is being recompiled - wait for
89 * completion */
90 pthread_cond_wait(&rec->cond2, &rec->mutex);
91
92 /* We can't guarantee the signal was for us.
93 * Since block_rec may have been removed while
94 * we were waiting on the condition, we cannot
95 * check block_rec->compiling again. The best
96 * thing is just to restart the function. */
97 return false;
98 }
99
100 /* Block is not yet being processed - remove it from the list */
101 slist_remove(&rec->slist, &block_rec->slist);
102 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
103 sizeof(*block_rec), block_rec);
104
105 return true;
106}
107
108static void lightrec_cancel_list(struct recompiler *rec)
109{
110 struct block_rec *block_rec;
ba3814c1 111 struct slist_elm *elm, *head = &rec->slist;
d8b04acd 112
ba3814c1
PC
113 for (elm = slist_first(head); elm; elm = slist_first(head)) {
114 block_rec = container_of(elm, struct block_rec, slist);
d8b04acd
PC
115 lightrec_cancel_block_rec(rec, block_rec);
116 }
d8b04acd
PC
117}
118
119static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d)
120{
121 struct recompiler *rec = d;
122
ba3814c1
PC
123 lightrec_remove_outdated_blocks(state->block_cache, NULL);
124 rec->must_flush = false;
d8b04acd
PC
125}
126
98fa08a5
PC
127static void lightrec_compile_list(struct recompiler *rec,
128 struct recompiler_thd *thd)
d16005f8 129{
a59e5536 130 struct block_rec *block_rec;
d16005f8
PC
131 struct block *block;
132 int ret;
133
a5a6f7b8 134 while (!!(block_rec = lightrec_get_best_elm(&rec->slist))) {
98fa08a5 135 block_rec->compiling = true;
a59e5536 136 block = block_rec->block;
d16005f8
PC
137
138 pthread_mutex_unlock(&rec->mutex);
139
ba3814c1 140 if (likely(!block_has_flag(block, BLOCK_IS_DEAD))) {
98fa08a5 141 ret = lightrec_compile_block(thd->cstate, block);
d8b04acd
PC
142 if (ret == -ENOMEM) {
143 /* Code buffer is full. Request the reaper to
144 * flush it. */
145
146 pthread_mutex_lock(&rec->mutex);
ba3814c1
PC
147 block_rec->compiling = false;
148 pthread_cond_broadcast(&rec->cond2);
149
d8b04acd 150 if (!rec->must_flush) {
ba3814c1
PC
151 rec->must_flush = true;
152 lightrec_cancel_list(rec);
153
d8b04acd
PC
154 lightrec_reaper_add(rec->state->reaper,
155 lightrec_flush_code_buffer,
156 rec);
d8b04acd
PC
157 }
158 return;
159 }
160
98fa08a5 161 if (ret) {
f5ee77ca 162 pr_err("Unable to compile block at "PC_FMT": %d\n",
98fa08a5
PC
163 block->pc, ret);
164 }
d16005f8
PC
165 }
166
167 pthread_mutex_lock(&rec->mutex);
168
a5a6f7b8 169 slist_remove(&rec->slist, &block_rec->slist);
d16005f8 170 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536 171 sizeof(*block_rec), block_rec);
ba3814c1 172 pthread_cond_broadcast(&rec->cond2);
d16005f8 173 }
d16005f8
PC
174}
175
176static void * lightrec_recompiler_thd(void *d)
177{
98fa08a5
PC
178 struct recompiler_thd *thd = d;
179 struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8
PC
180
181 pthread_mutex_lock(&rec->mutex);
182
ea17432f 183 while (!rec->stop) {
d16005f8
PC
184 do {
185 pthread_cond_wait(&rec->cond, &rec->mutex);
186
a59e5536 187 if (rec->stop)
188 goto out_unlock;
d16005f8 189
a59e5536 190 } while (slist_empty(&rec->slist));
d16005f8 191
98fa08a5 192 lightrec_compile_list(rec, thd);
ea17432f 193 }
a59e5536 194
195out_unlock:
196 pthread_mutex_unlock(&rec->mutex);
197 return NULL;
d16005f8
PC
198}
199
200struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
201{
202 struct recompiler *rec;
98fa08a5 203 unsigned int i, nb_recs, nb_cpus;
d16005f8
PC
204 int ret;
205
98fa08a5
PC
206 nb_cpus = get_processors_count();
207 nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
208
209 rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
210 + nb_recs * sizeof(*rec->thds));
d16005f8
PC
211 if (!rec) {
212 pr_err("Cannot create recompiler: Out of memory\n");
213 return NULL;
214 }
215
98fa08a5
PC
216 for (i = 0; i < nb_recs; i++) {
217 rec->thds[i].tid = i;
218 rec->thds[i].cstate = NULL;
219 }
220
221 for (i = 0; i < nb_recs; i++) {
222 rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e 223 if (!rec->thds[i].cstate) {
98fa08a5
PC
224 pr_err("Cannot create recompiler: Out of memory\n");
225 goto err_free_cstates;
226 }
227 }
228
d16005f8
PC
229 rec->state = state;
230 rec->stop = false;
d8b04acd 231 rec->must_flush = false;
98fa08a5 232 rec->nb_recs = nb_recs;
a59e5536 233 slist_init(&rec->slist);
d16005f8
PC
234
235 ret = pthread_cond_init(&rec->cond, NULL);
236 if (ret) {
237 pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5 238 goto err_free_cstates;
d16005f8
PC
239 }
240
98fa08a5 241 ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8 242 if (ret) {
98fa08a5 243 pr_err("Cannot init cond variable: %d\n", ret);
d16005f8
PC
244 goto err_cnd_destroy;
245 }
246
d8b04acd
PC
247 ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
248 if (ret) {
249 pr_err("Cannot init alloc mutex variable: %d\n", ret);
250 goto err_cnd2_destroy;
251 }
252
98fa08a5 253 ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8 254 if (ret) {
98fa08a5 255 pr_err("Cannot init mutex variable: %d\n", ret);
d8b04acd 256 goto err_alloc_mtx_destroy;
d16005f8
PC
257 }
258
98fa08a5
PC
259 for (i = 0; i < nb_recs; i++) {
260 ret = pthread_create(&rec->thds[i].thd, NULL,
261 lightrec_recompiler_thd, &rec->thds[i]);
262 if (ret) {
263 pr_err("Cannot create recompiler thread: %d\n", ret);
264 /* TODO: Handle cleanup properly */
265 goto err_mtx_destroy;
266 }
267 }
268
269 pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
270
d16005f8
PC
271 return rec;
272
273err_mtx_destroy:
274 pthread_mutex_destroy(&rec->mutex);
d8b04acd
PC
275err_alloc_mtx_destroy:
276 pthread_mutex_destroy(&rec->alloc_mutex);
98fa08a5
PC
277err_cnd2_destroy:
278 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
279err_cnd_destroy:
280 pthread_cond_destroy(&rec->cond);
98fa08a5
PC
281err_free_cstates:
282 for (i = 0; i < nb_recs; i++) {
283 if (rec->thds[i].cstate)
284 lightrec_free_cstate(rec->thds[i].cstate);
285 }
d16005f8
PC
286 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
287 return NULL;
288}
289
290void lightrec_free_recompiler(struct recompiler *rec)
291{
98fa08a5
PC
292 unsigned int i;
293
d16005f8
PC
294 rec->stop = true;
295
296 /* Stop the thread */
297 pthread_mutex_lock(&rec->mutex);
98fa08a5 298 pthread_cond_broadcast(&rec->cond);
d8b04acd 299 lightrec_cancel_list(rec);
d16005f8 300 pthread_mutex_unlock(&rec->mutex);
98fa08a5
PC
301
302 for (i = 0; i < rec->nb_recs; i++)
303 pthread_join(rec->thds[i].thd, NULL);
304
305 for (i = 0; i < rec->nb_recs; i++)
306 lightrec_free_cstate(rec->thds[i].cstate);
d16005f8
PC
307
308 pthread_mutex_destroy(&rec->mutex);
d8b04acd 309 pthread_mutex_destroy(&rec->alloc_mutex);
d16005f8 310 pthread_cond_destroy(&rec->cond);
98fa08a5 311 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
312 lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
313}
314
315int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
316{
a5a6f7b8 317 struct slist_elm *elm;
a59e5536 318 struct block_rec *block_rec;
a5a6f7b8 319 u32 pc1, pc2;
a59e5536 320 int ret = 0;
d16005f8
PC
321
322 pthread_mutex_lock(&rec->mutex);
323
d8b04acd
PC
324 /* If the recompiler must flush the code cache, we can't add the new
325 * job. It will be re-added next time the block's address is jumped to
326 * again. */
327 if (rec->must_flush)
328 goto out_unlock;
329
a59e5536 330 /* If the block is marked as dead, don't compile it, it will be removed
331 * as soon as it's safe. */
ba3814c1 332 if (block_has_flag(block, BLOCK_IS_DEAD))
a59e5536 333 goto out_unlock;
334
a5a6f7b8 335 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
a59e5536 336 block_rec = container_of(elm, struct block_rec, slist);
337
d16005f8 338 if (block_rec->block == block) {
a5a6f7b8
PC
339 /* The block to compile is already in the queue -
340 * increment its counter to increase its priority */
341 block_rec->requests++;
342 goto out_unlock;
343 }
d16005f8 344
a5a6f7b8
PC
345 pc1 = kunseg(block_rec->block->pc);
346 pc2 = kunseg(block->pc);
347 if (pc2 >= pc1 && pc2 < pc1 + block_rec->block->nb_ops * 4) {
348 /* The block we want to compile is already covered by
349 * another one in the queue - increment its counter to
350 * increase its priority */
351 block_rec->requests++;
a59e5536 352 goto out_unlock;
d16005f8
PC
353 }
354 }
355
356 /* By the time this function was called, the block has been recompiled
357 * and ins't in the wait list anymore. Just return here. */
ba3814c1 358 if (block->function && !block_has_flag(block, BLOCK_SHOULD_RECOMPILE))
a59e5536 359 goto out_unlock;
d16005f8
PC
360
361 block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
362 sizeof(*block_rec));
363 if (!block_rec) {
a59e5536 364 ret = -ENOMEM;
365 goto out_unlock;
d16005f8
PC
366 }
367
f5ee77ca 368 pr_debug("Adding block "PC_FMT" to recompiler\n", block->pc);
d16005f8
PC
369
370 block_rec->block = block;
98fa08a5 371 block_rec->compiling = false;
a5a6f7b8 372 block_rec->requests = 1;
a59e5536 373
374 elm = &rec->slist;
375
a5a6f7b8 376 /* Push the new entry to the front of the queue */
a59e5536 377 slist_append(elm, &block_rec->slist);
d16005f8
PC
378
379 /* Signal the thread */
380 pthread_cond_signal(&rec->cond);
d16005f8 381
a59e5536 382out_unlock:
383 pthread_mutex_unlock(&rec->mutex);
98fa08a5 384
a59e5536 385 return ret;
d16005f8
PC
386}
387
388void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
389{
390 struct block_rec *block_rec;
a59e5536 391 struct slist_elm *elm;
d16005f8
PC
392
393 pthread_mutex_lock(&rec->mutex);
394
98fa08a5
PC
395 while (true) {
396 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
397 block_rec = container_of(elm, struct block_rec, slist);
a59e5536 398
d8b04acd
PC
399 if (block_rec->block == block) {
400 if (lightrec_cancel_block_rec(rec, block_rec))
401 goto out_unlock;
98fa08a5 402
98fa08a5 403 break;
d16005f8 404 }
98fa08a5 405 }
d16005f8 406
98fa08a5 407 if (!elm)
d16005f8 408 break;
d16005f8
PC
409 }
410
98fa08a5 411out_unlock:
d16005f8
PC
412 pthread_mutex_unlock(&rec->mutex);
413}
414
98fa08a5
PC
415void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
416 struct block *block, u32 *pc)
d16005f8 417{
ba3814c1 418 u8 old_flags;
d16005f8 419
98fa08a5
PC
420 /* There's no point in running the first pass if the block will never
421 * be compiled. Let the main loop run the interpreter instead. */
ba3814c1 422 if (block_has_flag(block, BLOCK_NEVER_COMPILE))
98fa08a5
PC
423 return NULL;
424
ba3814c1
PC
425 /* The block is marked as dead, and will be removed the next time the
426 * reaper is run. In the meantime, the old function can still be
427 * executed. */
428 if (block_has_flag(block, BLOCK_IS_DEAD))
429 return block->function;
430
98fa08a5
PC
431 /* If the block is already fully tagged, there is no point in running
432 * the first pass. Request a recompilation of the block, and maybe the
433 * interpreter will run the block in the meantime. */
ba3814c1 434 if (block_has_flag(block, BLOCK_FULLY_TAGGED))
98fa08a5
PC
435 lightrec_recompiler_add(state->rec, block);
436
d16005f8 437 if (likely(block->function)) {
ba3814c1
PC
438 if (block_has_flag(block, BLOCK_FULLY_TAGGED)) {
439 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8 440
ba3814c1 441 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
f5ee77ca 442 pr_debug("Block "PC_FMT" is fully tagged"
d16005f8
PC
443 " - free opcode list\n", block->pc);
444
445 /* The block was already compiled but the opcode list
446 * didn't get freed yet - do it now */
ba3814c1 447 lightrec_free_opcode_list(state, block->opcode_list);
d16005f8
PC
448 }
449 }
450
451 return block->function;
452 }
453
454 /* Mark the opcode list as freed, so that the threaded compiler won't
455 * free it while we're using it in the interpreter. */
ba3814c1 456 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8
PC
457
458 /* Block wasn't compiled yet - run the interpreter */
98fa08a5 459 *pc = lightrec_emulate_block(state, block, *pc);
d16005f8 460
ba3814c1
PC
461 if (!(old_flags & BLOCK_NO_OPCODE_LIST))
462 block_clear_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8
PC
463
464 /* The block got compiled while the interpreter was running.
465 * We can free the opcode list now. */
ba3814c1
PC
466 if (block->function && block_has_flag(block, BLOCK_FULLY_TAGGED)) {
467 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
d16005f8 468
ba3814c1 469 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
f5ee77ca 470 pr_debug("Block "PC_FMT" is fully tagged"
ba3814c1
PC
471 " - free opcode list\n", block->pc);
472
473 lightrec_free_opcode_list(state, block->opcode_list);
474 }
d16005f8
PC
475 }
476
477 return NULL;
478}
d8b04acd
PC
479
480void lightrec_code_alloc_lock(struct lightrec_state *state)
481{
482 pthread_mutex_lock(&state->rec->alloc_mutex);
483}
484
485void lightrec_code_alloc_unlock(struct lightrec_state *state)
486{
487 pthread_mutex_unlock(&state->rec->alloc_mutex);
488}