1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "interpreter.h"
9 #include "lightrec-private.h"
10 #include "memmanager.h"
15 #include <stdatomic.h>
25 struct slist_elm slist;
26 unsigned int requests;
30 struct recompiler_thd {
31 struct lightrec_cstate *cstate;
37 struct lightrec_state *state;
40 pthread_mutex_t mutex;
41 bool stop, pause, must_flush;
42 struct slist_elm slist;
44 pthread_mutex_t alloc_mutex;
47 struct recompiler_thd thds[];
50 static unsigned int get_processors_count(void)
54 #if defined(PTW32_VERSION)
55 nb = pthread_num_processors_np();
56 #elif defined(__APPLE__) || defined(__FreeBSD__)
58 size_t size = sizeof(count);
60 nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
61 #elif defined(_SC_NPROCESSORS_ONLN)
62 nb = (int)sysconf(_SC_NPROCESSORS_ONLN);
65 return nb < 1 ? 1 : nb;
68 static struct block_rec * lightrec_get_best_elm(struct slist_elm *head)
70 struct block_rec *block_rec, *best = NULL;
71 struct slist_elm *elm;
73 for (elm = slist_first(head); elm; elm = elm->next) {
74 block_rec = container_of(elm, struct block_rec, slist);
76 if (!block_rec->compiling
77 && (!best || block_rec->requests > best->requests))
84 static bool lightrec_cancel_block_rec(struct recompiler *rec,
85 struct block_rec *block_rec)
87 if (block_rec->compiling) {
88 /* Block is being recompiled - wait for
90 pthread_cond_wait(&rec->cond2, &rec->mutex);
92 /* We can't guarantee the signal was for us.
93 * Since block_rec may have been removed while
94 * we were waiting on the condition, we cannot
95 * check block_rec->compiling again. The best
96 * thing is just to restart the function. */
100 /* Block is not yet being processed - remove it from the list */
101 slist_remove(&rec->slist, &block_rec->slist);
102 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
103 sizeof(*block_rec), block_rec);
108 static void lightrec_cancel_list(struct recompiler *rec)
110 struct block_rec *block_rec;
111 struct slist_elm *elm, *head = &rec->slist;
113 for (elm = slist_first(head); elm; elm = slist_first(head)) {
114 block_rec = container_of(elm, struct block_rec, slist);
115 lightrec_cancel_block_rec(rec, block_rec);
119 static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d)
121 struct recompiler *rec = d;
123 lightrec_remove_outdated_blocks(state->block_cache, NULL);
124 rec->must_flush = false;
127 static void lightrec_compile_list(struct recompiler *rec,
128 struct recompiler_thd *thd)
130 struct block_rec *block_rec;
134 while (!rec->pause &&
135 !!(block_rec = lightrec_get_best_elm(&rec->slist))) {
136 block_rec->compiling = true;
137 block = block_rec->block;
139 pthread_mutex_unlock(&rec->mutex);
141 if (likely(!block_has_flag(block, BLOCK_IS_DEAD))) {
142 ret = lightrec_compile_block(thd->cstate, block);
143 if (ret == -ENOMEM) {
144 /* Code buffer is full. Request the reaper to
147 pthread_mutex_lock(&rec->mutex);
148 block_rec->compiling = false;
149 pthread_cond_broadcast(&rec->cond2);
151 if (!rec->must_flush) {
152 rec->must_flush = true;
153 lightrec_cancel_list(rec);
155 lightrec_reaper_add(rec->state->reaper,
156 lightrec_flush_code_buffer,
163 pr_err("Unable to compile block at "PC_FMT": %d\n",
168 pthread_mutex_lock(&rec->mutex);
170 slist_remove(&rec->slist, &block_rec->slist);
171 lightrec_free(rec->state, MEM_FOR_LIGHTREC,
172 sizeof(*block_rec), block_rec);
173 pthread_cond_broadcast(&rec->cond2);
177 static void * lightrec_recompiler_thd(void *d)
179 struct recompiler_thd *thd = d;
180 struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
182 pthread_mutex_lock(&rec->mutex);
186 pthread_cond_wait(&rec->cond, &rec->mutex);
191 } while (rec->pause || slist_empty(&rec->slist));
193 lightrec_compile_list(rec, thd);
197 pthread_mutex_unlock(&rec->mutex);
201 struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
203 struct recompiler *rec;
204 unsigned int i, nb_recs, nb_cpus;
207 nb_cpus = get_processors_count();
208 nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
210 rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
211 + nb_recs * sizeof(*rec->thds));
213 pr_err("Cannot create recompiler: Out of memory\n");
217 for (i = 0; i < nb_recs; i++) {
218 rec->thds[i].tid = i;
219 rec->thds[i].cstate = NULL;
222 for (i = 0; i < nb_recs; i++) {
223 rec->thds[i].cstate = lightrec_create_cstate(state);
224 if (!rec->thds[i].cstate) {
225 pr_err("Cannot create recompiler: Out of memory\n");
226 goto err_free_cstates;
233 rec->must_flush = false;
234 rec->nb_recs = nb_recs;
235 slist_init(&rec->slist);
237 ret = pthread_cond_init(&rec->cond, NULL);
239 pr_err("Cannot init cond variable: %d\n", ret);
240 goto err_free_cstates;
243 ret = pthread_cond_init(&rec->cond2, NULL);
245 pr_err("Cannot init cond variable: %d\n", ret);
246 goto err_cnd_destroy;
249 ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
251 pr_err("Cannot init alloc mutex variable: %d\n", ret);
252 goto err_cnd2_destroy;
255 ret = pthread_mutex_init(&rec->mutex, NULL);
257 pr_err("Cannot init mutex variable: %d\n", ret);
258 goto err_alloc_mtx_destroy;
261 for (i = 0; i < nb_recs; i++) {
262 ret = pthread_create(&rec->thds[i].thd, NULL,
263 lightrec_recompiler_thd, &rec->thds[i]);
265 pr_err("Cannot create recompiler thread: %d\n", ret);
266 /* TODO: Handle cleanup properly */
267 goto err_mtx_destroy;
271 pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
276 pthread_mutex_destroy(&rec->mutex);
277 err_alloc_mtx_destroy:
278 pthread_mutex_destroy(&rec->alloc_mutex);
280 pthread_cond_destroy(&rec->cond2);
282 pthread_cond_destroy(&rec->cond);
284 for (i = 0; i < nb_recs; i++) {
285 if (rec->thds[i].cstate)
286 lightrec_free_cstate(rec->thds[i].cstate);
288 lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
292 void lightrec_free_recompiler(struct recompiler *rec)
298 /* Stop the thread */
299 pthread_mutex_lock(&rec->mutex);
300 pthread_cond_broadcast(&rec->cond);
301 lightrec_cancel_list(rec);
302 pthread_mutex_unlock(&rec->mutex);
304 for (i = 0; i < rec->nb_recs; i++)
305 pthread_join(rec->thds[i].thd, NULL);
307 for (i = 0; i < rec->nb_recs; i++)
308 lightrec_free_cstate(rec->thds[i].cstate);
310 pthread_mutex_destroy(&rec->mutex);
311 pthread_mutex_destroy(&rec->alloc_mutex);
312 pthread_cond_destroy(&rec->cond);
313 pthread_cond_destroy(&rec->cond2);
314 lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
317 int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
319 struct slist_elm *elm;
320 struct block_rec *block_rec;
324 pthread_mutex_lock(&rec->mutex);
326 /* If the recompiler must flush the code cache, we can't add the new
327 * job. It will be re-added next time the block's address is jumped to
332 /* If the block is marked as dead, don't compile it, it will be removed
333 * as soon as it's safe. */
334 if (block_has_flag(block, BLOCK_IS_DEAD))
337 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
338 block_rec = container_of(elm, struct block_rec, slist);
340 if (block_rec->block == block) {
341 /* The block to compile is already in the queue -
342 * increment its counter to increase its priority */
343 block_rec->requests++;
347 pc1 = kunseg(block_rec->block->pc);
348 pc2 = kunseg(block->pc);
349 if (pc2 >= pc1 && pc2 < pc1 + block_rec->block->nb_ops * 4) {
350 /* The block we want to compile is already covered by
351 * another one in the queue - increment its counter to
352 * increase its priority */
353 block_rec->requests++;
358 /* By the time this function was called, the block has been recompiled
359 * and ins't in the wait list anymore. Just return here. */
360 if (block->function && !block_has_flag(block, BLOCK_SHOULD_RECOMPILE))
363 block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
370 pr_debug("Adding block "PC_FMT" to recompiler\n", block->pc);
372 block_rec->block = block;
373 block_rec->compiling = false;
374 block_rec->requests = 1;
378 /* Push the new entry to the front of the queue */
379 slist_append(elm, &block_rec->slist);
381 /* Signal the thread */
382 pthread_cond_signal(&rec->cond);
385 pthread_mutex_unlock(&rec->mutex);
390 void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
392 struct block_rec *block_rec;
393 struct slist_elm *elm;
395 pthread_mutex_lock(&rec->mutex);
398 for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
399 block_rec = container_of(elm, struct block_rec, slist);
401 if (block_rec->block == block) {
402 if (lightrec_cancel_block_rec(rec, block_rec))
414 pthread_mutex_unlock(&rec->mutex);
417 void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
418 struct block *block, u32 *pc)
422 /* There's no point in running the first pass if the block will never
423 * be compiled. Let the main loop run the interpreter instead. */
424 if (block_has_flag(block, BLOCK_NEVER_COMPILE))
427 /* The block is marked as dead, and will be removed the next time the
428 * reaper is run. In the meantime, the old function can still be
430 if (block_has_flag(block, BLOCK_IS_DEAD))
431 return block->function;
433 /* If the block is already fully tagged, there is no point in running
434 * the first pass. Request a recompilation of the block, and maybe the
435 * interpreter will run the block in the meantime. */
436 if (block_has_flag(block, BLOCK_FULLY_TAGGED))
437 lightrec_recompiler_add(state->rec, block);
439 if (likely(block->function)) {
440 if (block_has_flag(block, BLOCK_FULLY_TAGGED)) {
441 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
443 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
444 pr_debug("Block "PC_FMT" is fully tagged"
445 " - free opcode list\n", block->pc);
447 /* The block was already compiled but the opcode list
448 * didn't get freed yet - do it now */
449 lightrec_free_opcode_list(state, block->opcode_list);
453 return block->function;
456 /* Mark the opcode list as freed, so that the threaded compiler won't
457 * free it while we're using it in the interpreter. */
458 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
460 /* Block wasn't compiled yet - run the interpreter */
461 *pc = lightrec_emulate_block(state, block, *pc);
463 if (!(old_flags & BLOCK_NO_OPCODE_LIST))
464 block_clear_flags(block, BLOCK_NO_OPCODE_LIST);
466 /* The block got compiled while the interpreter was running.
467 * We can free the opcode list now. */
468 if (block->function && block_has_flag(block, BLOCK_FULLY_TAGGED)) {
469 old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST);
471 if (!(old_flags & BLOCK_NO_OPCODE_LIST)) {
472 pr_debug("Block "PC_FMT" is fully tagged"
473 " - free opcode list\n", block->pc);
475 lightrec_free_opcode_list(state, block->opcode_list);
482 void lightrec_code_alloc_lock(struct lightrec_state *state)
484 pthread_mutex_lock(&state->rec->alloc_mutex);
487 void lightrec_code_alloc_unlock(struct lightrec_state *state)
489 pthread_mutex_unlock(&state->rec->alloc_mutex);
492 void lightrec_recompiler_pause(struct recompiler *rec)
496 pthread_mutex_lock(&rec->mutex);
497 pthread_cond_broadcast(&rec->cond);
498 lightrec_cancel_list(rec);
499 pthread_mutex_unlock(&rec->mutex);
502 void lightrec_recompiler_unpause(struct recompiler *rec)