gpu_neon: don't include vector_ops.h in the main header
[pcsx_rearmed.git] / deps / lightrec / recompiler.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC
4 */
5
d8b04acd 6#include "blockcache.h"
d16005f8
PC
7#include "debug.h"
8#include "interpreter.h"
9#include "lightrec-private.h"
10#include "memmanager.h"
d8b04acd 11#include "reaper.h"
a59e5536 12#include "slist.h"
d16005f8
PC
13
14#include <errno.h>
15#include <stdatomic.h>
16#include <stdbool.h>
17#include <stdlib.h>
18#include <pthread.h>
98fa08a5
PC
19#ifdef __linux__
20#include <unistd.h>
21#endif
d16005f8
PC
22
23struct block_rec {
24 struct block *block;
a59e5536 25 struct slist_elm slist;
98fa08a5
PC
26 bool compiling;
27};
28
29struct recompiler_thd {
30 struct lightrec_cstate *cstate;
31 unsigned int tid;
32 pthread_t thd;
d16005f8
PC
33};
34
35struct recompiler {
36 struct lightrec_state *state;
d16005f8 37 pthread_cond_t cond;
98fa08a5 38 pthread_cond_t cond2;
d16005f8 39 pthread_mutex_t mutex;
d8b04acd 40 bool stop, must_flush;
a59e5536 41 struct slist_elm slist;
98fa08a5 42
d8b04acd
PC
43 pthread_mutex_t alloc_mutex;
44
98fa08a5
PC
45 unsigned int nb_recs;
46 struct recompiler_thd thds[];
d16005f8
PC
47};
48
98fa08a5
PC
49static unsigned int get_processors_count(void)
50{
11357fef 51 unsigned int nb = 1;
98fa08a5
PC
52
53#if defined(PTW32_VERSION)
54 nb = pthread_num_processors_np();
55#elif defined(__APPLE__) || defined(__FreeBSD__)
56 int count;
57 size_t size = sizeof(count);
58
59 nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
11357fef 60#elif defined(_SC_NPROCESSORS_ONLN)
98fa08a5
PC
61 nb = sysconf(_SC_NPROCESSORS_ONLN);
62#endif
63
64 return nb < 1 ? 1 : nb;
65}
66
67static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
68{
69 struct block_rec *block_rec;
70 struct slist_elm *elm;
71
72 for (elm = slist_first(head); elm; elm = elm->next) {
73 block_rec = container_of(elm, struct block_rec, slist);
74
75 if (!block_rec->compiling)
76 return elm;
77 }
78
79 return NULL;
80}
81
d8b04acd
PC
82static bool lightrec_cancel_block_rec(struct recompiler *rec,
83 struct block_rec *block_rec)
84{
85 if (block_rec->compiling) {
86 /* Block is being recompiled - wait for
87 * completion */
88 pthread_cond_wait(&rec->cond2, &rec->mutex);
89
90 /* We can't guarantee the signal was for us.
91 * Since block_rec may have been removed while
92 * we were waiting on the condition, we cannot
93 * check block_rec->compiling again. The best
94 * thing is just to restart the function. */
95 return false;
96 }
97
98 /* Block is not yet being processed - remove it from the list */
99 slist_remove(&rec->slist, &block_rec->slist);
100 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
101 sizeof(*block_rec), block_rec);
102
103 return true;
104}
105
106static void lightrec_cancel_list(struct recompiler *rec)
107{
108 struct block_rec *block_rec;
109 struct slist_elm *next;
110
111 while (!!(next = lightrec_get_first_elm(&rec->slist))) {
112 block_rec = container_of(next, struct block_rec, slist);
113
114 lightrec_cancel_block_rec(rec, block_rec);
115 }
116
117 pthread_cond_broadcast(&rec->cond2);
118}
119
120static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d)
121{
122 struct recompiler *rec = d;
123
124 pthread_mutex_lock(&rec->mutex);
125
126 if (rec->must_flush) {
127 lightrec_remove_outdated_blocks(state->block_cache, NULL);
128 rec->must_flush = false;
129 }
130
131 pthread_mutex_unlock(&rec->mutex);
132}
133
98fa08a5
PC
134static void lightrec_compile_list(struct recompiler *rec,
135 struct recompiler_thd *thd)
d16005f8 136{
a59e5536 137 struct block_rec *block_rec;
138 struct slist_elm *next;
d16005f8
PC
139 struct block *block;
140 int ret;
141
98fa08a5 142 while (!!(next = lightrec_get_first_elm(&rec->slist))) {
a59e5536 143 block_rec = container_of(next, struct block_rec, slist);
98fa08a5 144 block_rec->compiling = true;
a59e5536 145 block = block_rec->block;
d16005f8
PC
146
147 pthread_mutex_unlock(&rec->mutex);
148
98fa08a5
PC
149 if (likely(!(block->flags & BLOCK_IS_DEAD))) {
150 ret = lightrec_compile_block(thd->cstate, block);
d8b04acd
PC
151 if (ret == -ENOMEM) {
152 /* Code buffer is full. Request the reaper to
153 * flush it. */
154
155 pthread_mutex_lock(&rec->mutex);
156 if (!rec->must_flush) {
157 lightrec_reaper_add(rec->state->reaper,
158 lightrec_flush_code_buffer,
159 rec);
160 lightrec_cancel_list(rec);
161 rec->must_flush = true;
162 }
163 return;
164 }
165
98fa08a5
PC
166 if (ret) {
167 pr_err("Unable to compile block at PC 0x%x: %d\n",
168 block->pc, ret);
169 }
d16005f8
PC
170 }
171
172 pthread_mutex_lock(&rec->mutex);
173
a59e5536 174 slist_remove(&rec->slist, next);
d16005f8 175 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536 176 sizeof(*block_rec), block_rec);
98fa08a5 177 pthread_cond_signal(&rec->cond2);
d16005f8 178 }
d16005f8
PC
179}
180
181static void * lightrec_recompiler_thd(void *d)
182{
98fa08a5
PC
183 struct recompiler_thd *thd = d;
184 struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8
PC
185
186 pthread_mutex_lock(&rec->mutex);
187
ea17432f 188 while (!rec->stop) {
d16005f8
PC
189 do {
190 pthread_cond_wait(&rec->cond, &rec->mutex);
191
a59e5536 192 if (rec->stop)
193 goto out_unlock;
d16005f8 194
a59e5536 195 } while (slist_empty(&rec->slist));
d16005f8 196
98fa08a5 197 lightrec_compile_list(rec, thd);
ea17432f 198 }
a59e5536 199
200out_unlock:
201 pthread_mutex_unlock(&rec->mutex);
202 return NULL;
d16005f8
PC
203}
204
205struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
206{
207 struct recompiler *rec;
98fa08a5 208 unsigned int i, nb_recs, nb_cpus;
d16005f8
PC
209 int ret;
210
98fa08a5
PC
211 nb_cpus = get_processors_count();
212 nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
213
214 rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
215 + nb_recs * sizeof(*rec->thds));
d16005f8
PC
216 if (!rec) {
217 pr_err("Cannot create recompiler: Out of memory\n");
218 return NULL;
219 }
220
98fa08a5
PC
221 for (i = 0; i < nb_recs; i++) {
222 rec->thds[i].tid = i;
223 rec->thds[i].cstate = NULL;
224 }
225
226 for (i = 0; i < nb_recs; i++) {
227 rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e 228 if (!rec->thds[i].cstate) {
98fa08a5
PC
229 pr_err("Cannot create recompiler: Out of memory\n");
230 goto err_free_cstates;
231 }
232 }
233
d16005f8
PC
234 rec->state = state;
235 rec->stop = false;
d8b04acd 236 rec->must_flush = false;
98fa08a5 237 rec->nb_recs = nb_recs;
a59e5536 238 slist_init(&rec->slist);
d16005f8
PC
239
240 ret = pthread_cond_init(&rec->cond, NULL);
241 if (ret) {
242 pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5 243 goto err_free_cstates;
d16005f8
PC
244 }
245
98fa08a5 246 ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8 247 if (ret) {
98fa08a5 248 pr_err("Cannot init cond variable: %d\n", ret);
d16005f8
PC
249 goto err_cnd_destroy;
250 }
251
d8b04acd
PC
252 ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
253 if (ret) {
254 pr_err("Cannot init alloc mutex variable: %d\n", ret);
255 goto err_cnd2_destroy;
256 }
257
98fa08a5 258 ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8 259 if (ret) {
98fa08a5 260 pr_err("Cannot init mutex variable: %d\n", ret);
d8b04acd 261 goto err_alloc_mtx_destroy;
d16005f8
PC
262 }
263
98fa08a5
PC
264 for (i = 0; i < nb_recs; i++) {
265 ret = pthread_create(&rec->thds[i].thd, NULL,
266 lightrec_recompiler_thd, &rec->thds[i]);
267 if (ret) {
268 pr_err("Cannot create recompiler thread: %d\n", ret);
269 /* TODO: Handle cleanup properly */
270 goto err_mtx_destroy;
271 }
272 }
273
274 pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
275
d16005f8
PC
276 return rec;
277
278err_mtx_destroy:
279 pthread_mutex_destroy(&rec->mutex);
d8b04acd
PC
280err_alloc_mtx_destroy:
281 pthread_mutex_destroy(&rec->alloc_mutex);
98fa08a5
PC
282err_cnd2_destroy:
283 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
284err_cnd_destroy:
285 pthread_cond_destroy(&rec->cond);
98fa08a5
PC
286err_free_cstates:
287 for (i = 0; i < nb_recs; i++) {
288 if (rec->thds[i].cstate)
289 lightrec_free_cstate(rec->thds[i].cstate);
290 }
d16005f8
PC
291 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
292 return NULL;
293}
294
295void lightrec_free_recompiler(struct recompiler *rec)
296{
98fa08a5
PC
297 unsigned int i;
298
d16005f8
PC
299 rec->stop = true;
300
301 /* Stop the thread */
302 pthread_mutex_lock(&rec->mutex);
98fa08a5 303 pthread_cond_broadcast(&rec->cond);
d8b04acd 304 lightrec_cancel_list(rec);
d16005f8 305 pthread_mutex_unlock(&rec->mutex);
98fa08a5
PC
306
307 for (i = 0; i < rec->nb_recs; i++)
308 pthread_join(rec->thds[i].thd, NULL);
309
310 for (i = 0; i < rec->nb_recs; i++)
311 lightrec_free_cstate(rec->thds[i].cstate);
d16005f8
PC
312
313 pthread_mutex_destroy(&rec->mutex);
d8b04acd 314 pthread_mutex_destroy(&rec->alloc_mutex);
d16005f8 315 pthread_cond_destroy(&rec->cond);
98fa08a5 316 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
317 lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
318}
319
320int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
321{
a59e5536 322 struct slist_elm *elm, *prev;
323 struct block_rec *block_rec;
324 int ret = 0;
d16005f8
PC
325
326 pthread_mutex_lock(&rec->mutex);
327
d8b04acd
PC
328 /* If the recompiler must flush the code cache, we can't add the new
329 * job. It will be re-added next time the block's address is jumped to
330 * again. */
331 if (rec->must_flush)
332 goto out_unlock;
333
a59e5536 334 /* If the block is marked as dead, don't compile it, it will be removed
335 * as soon as it's safe. */
336 if (block->flags & BLOCK_IS_DEAD)
337 goto out_unlock;
338
339 for (elm = slist_first(&rec->slist), prev = NULL; elm;
340 prev = elm, elm = elm->next) {
341 block_rec = container_of(elm, struct block_rec, slist);
342
d16005f8
PC
343 if (block_rec->block == block) {
344 /* The block to compile is already in the queue - bump
a59e5536 345 * it to the top of the list, unless the block is being
346 * recompiled. */
98fa08a5
PC
347 if (prev && !block_rec->compiling &&
348 !(block->flags & BLOCK_SHOULD_RECOMPILE)) {
a59e5536 349 slist_remove_next(prev);
350 slist_append(&rec->slist, elm);
d16005f8
PC
351 }
352
a59e5536 353 goto out_unlock;
d16005f8
PC
354 }
355 }
356
357 /* By the time this function was called, the block has been recompiled
358 * and ins't in the wait list anymore. Just return here. */
a59e5536 359 if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
360 goto out_unlock;
d16005f8
PC
361
362 block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
363 sizeof(*block_rec));
364 if (!block_rec) {
a59e5536 365 ret = -ENOMEM;
366 goto out_unlock;
d16005f8
PC
367 }
368
369 pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);
370
371 block_rec->block = block;
98fa08a5 372 block_rec->compiling = false;
a59e5536 373
374 elm = &rec->slist;
375
376 /* If the block is being recompiled, push it to the end of the queue;
377 * otherwise push it to the front of the queue. */
378 if (block->flags & BLOCK_SHOULD_RECOMPILE)
379 for (; elm->next; elm = elm->next);
380
381 slist_append(elm, &block_rec->slist);
d16005f8
PC
382
383 /* Signal the thread */
384 pthread_cond_signal(&rec->cond);
d16005f8 385
a59e5536 386out_unlock:
387 pthread_mutex_unlock(&rec->mutex);
98fa08a5 388
a59e5536 389 return ret;
d16005f8
PC
390}
391
392void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
393{
394 struct block_rec *block_rec;
a59e5536 395 struct slist_elm *elm;
d16005f8
PC
396
397 pthread_mutex_lock(&rec->mutex);
398
98fa08a5
PC
399 while (true) {
400 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
401 block_rec = container_of(elm, struct block_rec, slist);
a59e5536 402
d8b04acd
PC
403 if (block_rec->block == block) {
404 if (lightrec_cancel_block_rec(rec, block_rec))
405 goto out_unlock;
98fa08a5 406
98fa08a5 407 break;
d16005f8 408 }
98fa08a5 409 }
d16005f8 410
98fa08a5 411 if (!elm)
d16005f8 412 break;
d16005f8
PC
413 }
414
98fa08a5 415out_unlock:
d16005f8
PC
416 pthread_mutex_unlock(&rec->mutex);
417}
418
98fa08a5
PC
419void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
420 struct block *block, u32 *pc)
d16005f8
PC
421{
422 bool freed;
423
98fa08a5
PC
424 /* There's no point in running the first pass if the block will never
425 * be compiled. Let the main loop run the interpreter instead. */
426 if (block->flags & BLOCK_NEVER_COMPILE)
427 return NULL;
428
429 /* If the block is already fully tagged, there is no point in running
430 * the first pass. Request a recompilation of the block, and maybe the
431 * interpreter will run the block in the meantime. */
432 if (block->flags & BLOCK_FULLY_TAGGED)
433 lightrec_recompiler_add(state->rec, block);
434
d16005f8
PC
435 if (likely(block->function)) {
436 if (block->flags & BLOCK_FULLY_TAGGED) {
437 freed = atomic_flag_test_and_set(&block->op_list_freed);
438
439 if (!freed) {
440 pr_debug("Block PC 0x%08x is fully tagged"
441 " - free opcode list\n", block->pc);
442
443 /* The block was already compiled but the opcode list
444 * didn't get freed yet - do it now */
98fa08a5 445 lightrec_free_opcode_list(state, block);
d16005f8
PC
446 block->opcode_list = NULL;
447 }
448 }
449
450 return block->function;
451 }
452
453 /* Mark the opcode list as freed, so that the threaded compiler won't
454 * free it while we're using it in the interpreter. */
455 freed = atomic_flag_test_and_set(&block->op_list_freed);
456
457 /* Block wasn't compiled yet - run the interpreter */
98fa08a5 458 *pc = lightrec_emulate_block(state, block, *pc);
d16005f8
PC
459
460 if (!freed)
461 atomic_flag_clear(&block->op_list_freed);
462
463 /* The block got compiled while the interpreter was running.
464 * We can free the opcode list now. */
465 if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
466 !atomic_flag_test_and_set(&block->op_list_freed)) {
467 pr_debug("Block PC 0x%08x is fully tagged"
468 " - free opcode list\n", block->pc);
469
98fa08a5 470 lightrec_free_opcode_list(state, block);
d16005f8
PC
471 block->opcode_list = NULL;
472 }
473
474 return NULL;
475}
d8b04acd
PC
476
477void lightrec_code_alloc_lock(struct lightrec_state *state)
478{
479 pthread_mutex_lock(&state->rec->alloc_mutex);
480}
481
482void lightrec_code_alloc_unlock(struct lightrec_state *state)
483{
484 pthread_mutex_unlock(&state->rec->alloc_mutex);
485}