git subrepo pull --force deps/lightrec
[pcsx_rearmed.git] / deps / lightrec / recompiler.c
1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 /*
3  * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
4  */
5
6 #include "debug.h"
7 #include "interpreter.h"
8 #include "lightrec-private.h"
9 #include "memmanager.h"
10 #include "slist.h"
11
12 #include <errno.h>
13 #include <stdatomic.h>
14 #include <stdbool.h>
15 #include <stdlib.h>
16 #include <pthread.h>
17 #ifdef __linux__
18 #include <unistd.h>
19 #endif
20
21 struct block_rec {
22         struct block *block;
23         struct slist_elm slist;
24         bool compiling;
25 };
26
27 struct recompiler_thd {
28         struct lightrec_cstate *cstate;
29         unsigned int tid;
30         pthread_t thd;
31 };
32
33 struct recompiler {
34         struct lightrec_state *state;
35         pthread_cond_t cond;
36         pthread_cond_t cond2;
37         pthread_mutex_t mutex;
38         bool stop;
39         struct slist_elm slist;
40
41         unsigned int nb_recs;
42         struct recompiler_thd thds[];
43 };
44
45 static unsigned int get_processors_count(void)
46 {
47         unsigned int nb;
48
49 #if defined(PTW32_VERSION)
50         nb = pthread_num_processors_np();
51 #elif defined(__APPLE__) || defined(__FreeBSD__)
52         int count;
53         size_t size = sizeof(count);
54
55         nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
56 #elif defined(__linux__)
57         nb = sysconf(_SC_NPROCESSORS_ONLN);
58 #endif
59
60         return nb < 1 ? 1 : nb;
61 }
62
63 static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
64 {
65         struct block_rec *block_rec;
66         struct slist_elm *elm;
67
68         for (elm = slist_first(head); elm; elm = elm->next) {
69                 block_rec = container_of(elm, struct block_rec, slist);
70
71                 if (!block_rec->compiling)
72                         return elm;
73         }
74
75         return NULL;
76 }
77
78 static void lightrec_compile_list(struct recompiler *rec,
79                                   struct recompiler_thd *thd)
80 {
81         struct block_rec *block_rec;
82         struct slist_elm *next;
83         struct block *block;
84         int ret;
85
86         while (!!(next = lightrec_get_first_elm(&rec->slist))) {
87                 block_rec = container_of(next, struct block_rec, slist);
88                 block_rec->compiling = true;
89                 block = block_rec->block;
90
91                 pthread_mutex_unlock(&rec->mutex);
92
93                 if (likely(!(block->flags & BLOCK_IS_DEAD))) {
94                         ret = lightrec_compile_block(thd->cstate, block);
95                         if (ret) {
96                                 pr_err("Unable to compile block at PC 0x%x: %d\n",
97                                        block->pc, ret);
98                         }
99                 }
100
101                 pthread_mutex_lock(&rec->mutex);
102
103                 slist_remove(&rec->slist, next);
104                 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
105                               sizeof(*block_rec), block_rec);
106                 pthread_cond_signal(&rec->cond2);
107         }
108 }
109
110 static void * lightrec_recompiler_thd(void *d)
111 {
112         struct recompiler_thd *thd = d;
113         struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
114
115         pthread_mutex_lock(&rec->mutex);
116
117         while (!rec->stop) {
118                 do {
119                         pthread_cond_wait(&rec->cond, &rec->mutex);
120
121                         if (rec->stop)
122                                 goto out_unlock;
123
124                 } while (slist_empty(&rec->slist));
125
126                 lightrec_compile_list(rec, thd);
127         }
128
129 out_unlock:
130         pthread_mutex_unlock(&rec->mutex);
131         return NULL;
132 }
133
134 struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
135 {
136         struct recompiler *rec;
137         unsigned int i, nb_recs, nb_cpus;
138         int ret;
139
140         nb_cpus = get_processors_count();
141         nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
142
143         rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
144                               + nb_recs * sizeof(*rec->thds));
145         if (!rec) {
146                 pr_err("Cannot create recompiler: Out of memory\n");
147                 return NULL;
148         }
149
150         for (i = 0; i < nb_recs; i++) {
151                 rec->thds[i].tid = i;
152                 rec->thds[i].cstate = NULL;
153         }
154
155         for (i = 0; i < nb_recs; i++) {
156                 rec->thds[i].cstate = lightrec_create_cstate(state);
157                 if (!rec->state) {
158                         pr_err("Cannot create recompiler: Out of memory\n");
159                         goto err_free_cstates;
160                 }
161         }
162
163         rec->state = state;
164         rec->stop = false;
165         rec->nb_recs = nb_recs;
166         slist_init(&rec->slist);
167
168         ret = pthread_cond_init(&rec->cond, NULL);
169         if (ret) {
170                 pr_err("Cannot init cond variable: %d\n", ret);
171                 goto err_free_cstates;
172         }
173
174         ret = pthread_cond_init(&rec->cond2, NULL);
175         if (ret) {
176                 pr_err("Cannot init cond variable: %d\n", ret);
177                 goto err_cnd_destroy;
178         }
179
180         ret = pthread_mutex_init(&rec->mutex, NULL);
181         if (ret) {
182                 pr_err("Cannot init mutex variable: %d\n", ret);
183                 goto err_cnd2_destroy;
184         }
185
186         for (i = 0; i < nb_recs; i++) {
187                 ret = pthread_create(&rec->thds[i].thd, NULL,
188                                      lightrec_recompiler_thd, &rec->thds[i]);
189                 if (ret) {
190                         pr_err("Cannot create recompiler thread: %d\n", ret);
191                         /* TODO: Handle cleanup properly */
192                         goto err_mtx_destroy;
193                 }
194         }
195
196         pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
197
198         return rec;
199
200 err_mtx_destroy:
201         pthread_mutex_destroy(&rec->mutex);
202 err_cnd2_destroy:
203         pthread_cond_destroy(&rec->cond2);
204 err_cnd_destroy:
205         pthread_cond_destroy(&rec->cond);
206 err_free_cstates:
207         for (i = 0; i < nb_recs; i++) {
208                 if (rec->thds[i].cstate)
209                         lightrec_free_cstate(rec->thds[i].cstate);
210         }
211         lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
212         return NULL;
213 }
214
215 void lightrec_free_recompiler(struct recompiler *rec)
216 {
217         unsigned int i;
218
219         rec->stop = true;
220
221         /* Stop the thread */
222         pthread_mutex_lock(&rec->mutex);
223         pthread_cond_broadcast(&rec->cond);
224         pthread_mutex_unlock(&rec->mutex);
225
226         for (i = 0; i < rec->nb_recs; i++)
227                 pthread_join(rec->thds[i].thd, NULL);
228
229         for (i = 0; i < rec->nb_recs; i++)
230                 lightrec_free_cstate(rec->thds[i].cstate);
231
232         pthread_mutex_destroy(&rec->mutex);
233         pthread_cond_destroy(&rec->cond);
234         pthread_cond_destroy(&rec->cond2);
235         lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
236 }
237
238 int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
239 {
240         struct slist_elm *elm, *prev;
241         struct block_rec *block_rec;
242         int ret = 0;
243
244         pthread_mutex_lock(&rec->mutex);
245
246         /* If the block is marked as dead, don't compile it, it will be removed
247          * as soon as it's safe. */
248         if (block->flags & BLOCK_IS_DEAD)
249                 goto out_unlock;
250
251         for (elm = slist_first(&rec->slist), prev = NULL; elm;
252              prev = elm, elm = elm->next) {
253                 block_rec = container_of(elm, struct block_rec, slist);
254
255                 if (block_rec->block == block) {
256                         /* The block to compile is already in the queue - bump
257                          * it to the top of the list, unless the block is being
258                          * recompiled. */
259                         if (prev && !block_rec->compiling &&
260                             !(block->flags & BLOCK_SHOULD_RECOMPILE)) {
261                                 slist_remove_next(prev);
262                                 slist_append(&rec->slist, elm);
263                         }
264
265                         goto out_unlock;
266                 }
267         }
268
269         /* By the time this function was called, the block has been recompiled
270          * and ins't in the wait list anymore. Just return here. */
271         if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
272                 goto out_unlock;
273
274         block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
275                                     sizeof(*block_rec));
276         if (!block_rec) {
277                 ret = -ENOMEM;
278                 goto out_unlock;
279         }
280
281         pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);
282
283         block_rec->block = block;
284         block_rec->compiling = false;
285
286         elm = &rec->slist;
287
288         /* If the block is being recompiled, push it to the end of the queue;
289          * otherwise push it to the front of the queue. */
290         if (block->flags & BLOCK_SHOULD_RECOMPILE)
291                 for (; elm->next; elm = elm->next);
292
293         slist_append(elm, &block_rec->slist);
294
295         /* Signal the thread */
296         pthread_cond_signal(&rec->cond);
297
298 out_unlock:
299         pthread_mutex_unlock(&rec->mutex);
300
301         return ret;
302 }
303
304 void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
305 {
306         struct block_rec *block_rec;
307         struct slist_elm *elm;
308
309         pthread_mutex_lock(&rec->mutex);
310
311         while (true) {
312                 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
313                         block_rec = container_of(elm, struct block_rec, slist);
314
315                         if (block_rec->block != block)
316                                 continue;
317
318                         if (block_rec->compiling) {
319                                 /* Block is being recompiled - wait for
320                                  * completion */
321                                 pthread_cond_wait(&rec->cond2, &rec->mutex);
322
323                                 /* We can't guarantee the signal was for us.
324                                  * Since block_rec may have been removed while
325                                  * we were waiting on the condition, we cannot
326                                  * check block_rec->compiling again. The best
327                                  * thing is just to restart the function. */
328                                 break;
329                         } else {
330                                 /* Block is not yet being processed - remove it
331                                  * from the list */
332                                 slist_remove(&rec->slist, elm);
333                                 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
334                                               sizeof(*block_rec), block_rec);
335
336                                 goto out_unlock;
337                         }
338                 }
339
340                 if (!elm)
341                         break;
342         }
343
344 out_unlock:
345         pthread_mutex_unlock(&rec->mutex);
346 }
347
348 void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
349                                           struct block *block, u32 *pc)
350 {
351         bool freed;
352
353         /* There's no point in running the first pass if the block will never
354          * be compiled. Let the main loop run the interpreter instead. */
355         if (block->flags & BLOCK_NEVER_COMPILE)
356                 return NULL;
357
358         /* If the block is already fully tagged, there is no point in running
359          * the first pass. Request a recompilation of the block, and maybe the
360          * interpreter will run the block in the meantime. */
361         if (block->flags & BLOCK_FULLY_TAGGED)
362                 lightrec_recompiler_add(state->rec, block);
363
364         if (likely(block->function)) {
365                 if (block->flags & BLOCK_FULLY_TAGGED) {
366                         freed = atomic_flag_test_and_set(&block->op_list_freed);
367
368                         if (!freed) {
369                                 pr_debug("Block PC 0x%08x is fully tagged"
370                                          " - free opcode list\n", block->pc);
371
372                                 /* The block was already compiled but the opcode list
373                                  * didn't get freed yet - do it now */
374                                 lightrec_free_opcode_list(state, block);
375                                 block->opcode_list = NULL;
376                         }
377                 }
378
379                 return block->function;
380         }
381
382         /* Mark the opcode list as freed, so that the threaded compiler won't
383          * free it while we're using it in the interpreter. */
384         freed = atomic_flag_test_and_set(&block->op_list_freed);
385
386         /* Block wasn't compiled yet - run the interpreter */
387         *pc = lightrec_emulate_block(state, block, *pc);
388
389         if (!freed)
390                 atomic_flag_clear(&block->op_list_freed);
391
392         /* The block got compiled while the interpreter was running.
393          * We can free the opcode list now. */
394         if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
395             !atomic_flag_test_and_set(&block->op_list_freed)) {
396                 pr_debug("Block PC 0x%08x is fully tagged"
397                          " - free opcode list\n", block->pc);
398
399                 lightrec_free_opcode_list(state, block);
400                 block->opcode_list = NULL;
401         }
402
403         return NULL;
404 }