psxmem: Add support for Lightrec's custom mem init sequence
[pcsx_rearmed.git] / deps / lightrec / recompiler.c
CommitLineData
98fa08a5 1// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8 2/*
98fa08a5 3 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8
PC
4 */
5
6#include "debug.h"
7#include "interpreter.h"
8#include "lightrec-private.h"
9#include "memmanager.h"
a59e5536 10#include "slist.h"
d16005f8
PC
11
12#include <errno.h>
13#include <stdatomic.h>
14#include <stdbool.h>
15#include <stdlib.h>
16#include <pthread.h>
98fa08a5
PC
17#ifdef __linux__
18#include <unistd.h>
19#endif
d16005f8
PC
20
21struct block_rec {
22 struct block *block;
a59e5536 23 struct slist_elm slist;
98fa08a5
PC
24 bool compiling;
25};
26
27struct recompiler_thd {
28 struct lightrec_cstate *cstate;
29 unsigned int tid;
30 pthread_t thd;
d16005f8
PC
31};
32
33struct recompiler {
34 struct lightrec_state *state;
d16005f8 35 pthread_cond_t cond;
98fa08a5 36 pthread_cond_t cond2;
d16005f8
PC
37 pthread_mutex_t mutex;
38 bool stop;
a59e5536 39 struct slist_elm slist;
98fa08a5
PC
40
41 unsigned int nb_recs;
42 struct recompiler_thd thds[];
d16005f8
PC
43};
44
98fa08a5
PC
45static unsigned int get_processors_count(void)
46{
47 unsigned int nb;
48
49#if defined(PTW32_VERSION)
50 nb = pthread_num_processors_np();
51#elif defined(__APPLE__) || defined(__FreeBSD__)
52 int count;
53 size_t size = sizeof(count);
54
55 nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
56#elif defined(__linux__)
57 nb = sysconf(_SC_NPROCESSORS_ONLN);
58#endif
59
60 return nb < 1 ? 1 : nb;
61}
62
63static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
64{
65 struct block_rec *block_rec;
66 struct slist_elm *elm;
67
68 for (elm = slist_first(head); elm; elm = elm->next) {
69 block_rec = container_of(elm, struct block_rec, slist);
70
71 if (!block_rec->compiling)
72 return elm;
73 }
74
75 return NULL;
76}
77
78static void lightrec_compile_list(struct recompiler *rec,
79 struct recompiler_thd *thd)
d16005f8 80{
a59e5536 81 struct block_rec *block_rec;
82 struct slist_elm *next;
d16005f8
PC
83 struct block *block;
84 int ret;
85
98fa08a5 86 while (!!(next = lightrec_get_first_elm(&rec->slist))) {
a59e5536 87 block_rec = container_of(next, struct block_rec, slist);
98fa08a5 88 block_rec->compiling = true;
a59e5536 89 block = block_rec->block;
d16005f8
PC
90
91 pthread_mutex_unlock(&rec->mutex);
92
98fa08a5
PC
93 if (likely(!(block->flags & BLOCK_IS_DEAD))) {
94 ret = lightrec_compile_block(thd->cstate, block);
95 if (ret) {
96 pr_err("Unable to compile block at PC 0x%x: %d\n",
97 block->pc, ret);
98 }
d16005f8
PC
99 }
100
101 pthread_mutex_lock(&rec->mutex);
102
a59e5536 103 slist_remove(&rec->slist, next);
d16005f8 104 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536 105 sizeof(*block_rec), block_rec);
98fa08a5 106 pthread_cond_signal(&rec->cond2);
d16005f8 107 }
d16005f8
PC
108}
109
110static void * lightrec_recompiler_thd(void *d)
111{
98fa08a5
PC
112 struct recompiler_thd *thd = d;
113 struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8
PC
114
115 pthread_mutex_lock(&rec->mutex);
116
ea17432f 117 while (!rec->stop) {
d16005f8
PC
118 do {
119 pthread_cond_wait(&rec->cond, &rec->mutex);
120
a59e5536 121 if (rec->stop)
122 goto out_unlock;
d16005f8 123
a59e5536 124 } while (slist_empty(&rec->slist));
d16005f8 125
98fa08a5 126 lightrec_compile_list(rec, thd);
ea17432f 127 }
a59e5536 128
129out_unlock:
130 pthread_mutex_unlock(&rec->mutex);
131 return NULL;
d16005f8
PC
132}
133
134struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
135{
136 struct recompiler *rec;
98fa08a5 137 unsigned int i, nb_recs, nb_cpus;
d16005f8
PC
138 int ret;
139
98fa08a5
PC
140 nb_cpus = get_processors_count();
141 nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
142
143 rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
144 + nb_recs * sizeof(*rec->thds));
d16005f8
PC
145 if (!rec) {
146 pr_err("Cannot create recompiler: Out of memory\n");
147 return NULL;
148 }
149
98fa08a5
PC
150 for (i = 0; i < nb_recs; i++) {
151 rec->thds[i].tid = i;
152 rec->thds[i].cstate = NULL;
153 }
154
155 for (i = 0; i < nb_recs; i++) {
156 rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e 157 if (!rec->thds[i].cstate) {
98fa08a5
PC
158 pr_err("Cannot create recompiler: Out of memory\n");
159 goto err_free_cstates;
160 }
161 }
162
d16005f8
PC
163 rec->state = state;
164 rec->stop = false;
98fa08a5 165 rec->nb_recs = nb_recs;
a59e5536 166 slist_init(&rec->slist);
d16005f8
PC
167
168 ret = pthread_cond_init(&rec->cond, NULL);
169 if (ret) {
170 pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5 171 goto err_free_cstates;
d16005f8
PC
172 }
173
98fa08a5 174 ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8 175 if (ret) {
98fa08a5 176 pr_err("Cannot init cond variable: %d\n", ret);
d16005f8
PC
177 goto err_cnd_destroy;
178 }
179
98fa08a5 180 ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8 181 if (ret) {
98fa08a5
PC
182 pr_err("Cannot init mutex variable: %d\n", ret);
183 goto err_cnd2_destroy;
d16005f8
PC
184 }
185
98fa08a5
PC
186 for (i = 0; i < nb_recs; i++) {
187 ret = pthread_create(&rec->thds[i].thd, NULL,
188 lightrec_recompiler_thd, &rec->thds[i]);
189 if (ret) {
190 pr_err("Cannot create recompiler thread: %d\n", ret);
191 /* TODO: Handle cleanup properly */
192 goto err_mtx_destroy;
193 }
194 }
195
196 pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
197
d16005f8
PC
198 return rec;
199
200err_mtx_destroy:
201 pthread_mutex_destroy(&rec->mutex);
98fa08a5
PC
202err_cnd2_destroy:
203 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
204err_cnd_destroy:
205 pthread_cond_destroy(&rec->cond);
98fa08a5
PC
206err_free_cstates:
207 for (i = 0; i < nb_recs; i++) {
208 if (rec->thds[i].cstate)
209 lightrec_free_cstate(rec->thds[i].cstate);
210 }
d16005f8
PC
211 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
212 return NULL;
213}
214
215void lightrec_free_recompiler(struct recompiler *rec)
216{
98fa08a5
PC
217 unsigned int i;
218
d16005f8
PC
219 rec->stop = true;
220
221 /* Stop the thread */
222 pthread_mutex_lock(&rec->mutex);
98fa08a5 223 pthread_cond_broadcast(&rec->cond);
d16005f8 224 pthread_mutex_unlock(&rec->mutex);
98fa08a5
PC
225
226 for (i = 0; i < rec->nb_recs; i++)
227 pthread_join(rec->thds[i].thd, NULL);
228
229 for (i = 0; i < rec->nb_recs; i++)
230 lightrec_free_cstate(rec->thds[i].cstate);
d16005f8
PC
231
232 pthread_mutex_destroy(&rec->mutex);
233 pthread_cond_destroy(&rec->cond);
98fa08a5 234 pthread_cond_destroy(&rec->cond2);
d16005f8
PC
235 lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
236}
237
238int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
239{
a59e5536 240 struct slist_elm *elm, *prev;
241 struct block_rec *block_rec;
242 int ret = 0;
d16005f8
PC
243
244 pthread_mutex_lock(&rec->mutex);
245
a59e5536 246 /* If the block is marked as dead, don't compile it, it will be removed
247 * as soon as it's safe. */
248 if (block->flags & BLOCK_IS_DEAD)
249 goto out_unlock;
250
251 for (elm = slist_first(&rec->slist), prev = NULL; elm;
252 prev = elm, elm = elm->next) {
253 block_rec = container_of(elm, struct block_rec, slist);
254
d16005f8
PC
255 if (block_rec->block == block) {
256 /* The block to compile is already in the queue - bump
a59e5536 257 * it to the top of the list, unless the block is being
258 * recompiled. */
98fa08a5
PC
259 if (prev && !block_rec->compiling &&
260 !(block->flags & BLOCK_SHOULD_RECOMPILE)) {
a59e5536 261 slist_remove_next(prev);
262 slist_append(&rec->slist, elm);
d16005f8
PC
263 }
264
a59e5536 265 goto out_unlock;
d16005f8
PC
266 }
267 }
268
269 /* By the time this function was called, the block has been recompiled
270 * and ins't in the wait list anymore. Just return here. */
a59e5536 271 if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
272 goto out_unlock;
d16005f8
PC
273
274 block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
275 sizeof(*block_rec));
276 if (!block_rec) {
a59e5536 277 ret = -ENOMEM;
278 goto out_unlock;
d16005f8
PC
279 }
280
281 pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);
282
283 block_rec->block = block;
98fa08a5 284 block_rec->compiling = false;
a59e5536 285
286 elm = &rec->slist;
287
288 /* If the block is being recompiled, push it to the end of the queue;
289 * otherwise push it to the front of the queue. */
290 if (block->flags & BLOCK_SHOULD_RECOMPILE)
291 for (; elm->next; elm = elm->next);
292
293 slist_append(elm, &block_rec->slist);
d16005f8
PC
294
295 /* Signal the thread */
296 pthread_cond_signal(&rec->cond);
d16005f8 297
a59e5536 298out_unlock:
299 pthread_mutex_unlock(&rec->mutex);
98fa08a5 300
a59e5536 301 return ret;
d16005f8
PC
302}
303
304void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
305{
306 struct block_rec *block_rec;
a59e5536 307 struct slist_elm *elm;
d16005f8
PC
308
309 pthread_mutex_lock(&rec->mutex);
310
98fa08a5
PC
311 while (true) {
312 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
313 block_rec = container_of(elm, struct block_rec, slist);
a59e5536 314
98fa08a5
PC
315 if (block_rec->block != block)
316 continue;
317
318 if (block_rec->compiling) {
d16005f8
PC
319 /* Block is being recompiled - wait for
320 * completion */
98fa08a5
PC
321 pthread_cond_wait(&rec->cond2, &rec->mutex);
322
323 /* We can't guarantee the signal was for us.
324 * Since block_rec may have been removed while
325 * we were waiting on the condition, we cannot
326 * check block_rec->compiling again. The best
327 * thing is just to restart the function. */
328 break;
d16005f8
PC
329 } else {
330 /* Block is not yet being processed - remove it
331 * from the list */
a59e5536 332 slist_remove(&rec->slist, elm);
d16005f8
PC
333 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
334 sizeof(*block_rec), block_rec);
98fa08a5
PC
335
336 goto out_unlock;
d16005f8 337 }
98fa08a5 338 }
d16005f8 339
98fa08a5 340 if (!elm)
d16005f8 341 break;
d16005f8
PC
342 }
343
98fa08a5 344out_unlock:
d16005f8
PC
345 pthread_mutex_unlock(&rec->mutex);
346}
347
98fa08a5
PC
348void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
349 struct block *block, u32 *pc)
d16005f8
PC
350{
351 bool freed;
352
98fa08a5
PC
353 /* There's no point in running the first pass if the block will never
354 * be compiled. Let the main loop run the interpreter instead. */
355 if (block->flags & BLOCK_NEVER_COMPILE)
356 return NULL;
357
358 /* If the block is already fully tagged, there is no point in running
359 * the first pass. Request a recompilation of the block, and maybe the
360 * interpreter will run the block in the meantime. */
361 if (block->flags & BLOCK_FULLY_TAGGED)
362 lightrec_recompiler_add(state->rec, block);
363
d16005f8
PC
364 if (likely(block->function)) {
365 if (block->flags & BLOCK_FULLY_TAGGED) {
366 freed = atomic_flag_test_and_set(&block->op_list_freed);
367
368 if (!freed) {
369 pr_debug("Block PC 0x%08x is fully tagged"
370 " - free opcode list\n", block->pc);
371
372 /* The block was already compiled but the opcode list
373 * didn't get freed yet - do it now */
98fa08a5 374 lightrec_free_opcode_list(state, block);
d16005f8
PC
375 block->opcode_list = NULL;
376 }
377 }
378
379 return block->function;
380 }
381
382 /* Mark the opcode list as freed, so that the threaded compiler won't
383 * free it while we're using it in the interpreter. */
384 freed = atomic_flag_test_and_set(&block->op_list_freed);
385
386 /* Block wasn't compiled yet - run the interpreter */
98fa08a5 387 *pc = lightrec_emulate_block(state, block, *pc);
d16005f8
PC
388
389 if (!freed)
390 atomic_flag_clear(&block->op_list_freed);
391
392 /* The block got compiled while the interpreter was running.
393 * We can free the opcode list now. */
394 if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
395 !atomic_flag_test_and_set(&block->op_list_freed)) {
396 pr_debug("Block PC 0x%08x is fully tagged"
397 " - free opcode list\n", block->pc);
398
98fa08a5 399 lightrec_free_opcode_list(state, block);
d16005f8
PC
400 block->opcode_list = NULL;
401 }
402
403 return NULL;
404}