[pcsx_rearmed.git] / deps / lightrec / recompiler.c

// SPDX-License-Identifier: LGPL-2.1-or-later
/*
 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
 */

#include "blockcache.h"
#include "debug.h"
#include "interpreter.h"
#include "lightrec-private.h"
#include "memmanager.h"
#include "reaper.h"
#include "slist.h"

#include <errno.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdlib.h>
#include <pthread.h>
#ifdef __linux__
#include <unistd.h>
#endif

struct block_rec {
	struct block *block;
	struct slist_elm slist;
	bool compiling;
};

struct recompiler_thd {
	struct lightrec_cstate *cstate;
	unsigned int tid;
	pthread_t thd;
};

struct recompiler {
	struct lightrec_state *state;
	pthread_cond_t cond;
	pthread_cond_t cond2;
	pthread_mutex_t mutex;
	bool stop, must_flush;
	struct slist_elm slist;

	pthread_mutex_t alloc_mutex;

	unsigned int nb_recs;
	struct recompiler_thd thds[];
};

static unsigned int get_processors_count(void)
{
	unsigned int nb;

#if defined(PTW32_VERSION)
        nb = pthread_num_processors_np();
#elif defined(__APPLE__) || defined(__FreeBSD__)
        int count;
        size_t size = sizeof(count);

        nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
#elif defined(__linux__)
	nb = sysconf(_SC_NPROCESSORS_ONLN);
#endif

	return nb < 1 ? 1 : nb;
}

static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
{
	struct block_rec *block_rec;
	struct slist_elm *elm;

	for (elm = slist_first(head); elm; elm = elm->next) {
		block_rec = container_of(elm, struct block_rec, slist);

		if (!block_rec->compiling)
			return elm;
	}

	return NULL;
}

static bool lightrec_cancel_block_rec(struct recompiler *rec,
				      struct block_rec *block_rec)
{
	if (block_rec->compiling) {
		/* Block is being recompiled - wait for
		 * completion */
		pthread_cond_wait(&rec->cond2, &rec->mutex);

		/* We can't guarantee the signal was for us.
		 * Since block_rec may have been removed while
		 * we were waiting on the condition, we cannot
		 * check block_rec->compiling again. The best
		 * thing is just to restart the function. */
		return false;
	}

	/* Block is not yet being processed - remove it from the list */
	slist_remove(&rec->slist, &block_rec->slist);
	lightrec_free(rec->state, MEM_FOR_LIGHTREC,
		      sizeof(*block_rec), block_rec);

	return true;
}

static void lightrec_cancel_list(struct recompiler *rec)
{
	struct block_rec *block_rec;
	struct slist_elm *next;

	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
		block_rec = container_of(next, struct block_rec, slist);

		lightrec_cancel_block_rec(rec, block_rec);
	}

	pthread_cond_broadcast(&rec->cond2);
}

static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d)
{
	struct recompiler *rec = d;

	pthread_mutex_lock(&rec->mutex);

	if (rec->must_flush) {
		lightrec_remove_outdated_blocks(state->block_cache, NULL);
		rec->must_flush = false;
	}

	pthread_mutex_unlock(&rec->mutex);
}

static void lightrec_compile_list(struct recompiler *rec,
				  struct recompiler_thd *thd)
{
	struct block_rec *block_rec;
	struct slist_elm *next;
	struct block *block;
	int ret;

	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
		block_rec = container_of(next, struct block_rec, slist);
		block_rec->compiling = true;
		block = block_rec->block;

		pthread_mutex_unlock(&rec->mutex);

		if (likely(!(block->flags & BLOCK_IS_DEAD))) {
			ret = lightrec_compile_block(thd->cstate, block);
			if (ret == -ENOMEM) {
				/* Code buffer is full. Request the reaper to
				 * flush it. */

				pthread_mutex_lock(&rec->mutex);
				if (!rec->must_flush) {
					lightrec_reaper_add(rec->state->reaper,
							    lightrec_flush_code_buffer,
							    rec);
					lightrec_cancel_list(rec);
					rec->must_flush = true;
				}
				return;
			}

			if (ret) {
				pr_err("Unable to compile block at PC 0x%x: %d\n",
				       block->pc, ret);
			}
		}

		pthread_mutex_lock(&rec->mutex);

		slist_remove(&rec->slist, next);
		lightrec_free(rec->state, MEM_FOR_LIGHTREC,
			      sizeof(*block_rec), block_rec);
		pthread_cond_signal(&rec->cond2);
	}
}

static void * lightrec_recompiler_thd(void *d)
{
	struct recompiler_thd *thd = d;
	struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);

	pthread_mutex_lock(&rec->mutex);

	while (!rec->stop) {
		do {
			pthread_cond_wait(&rec->cond, &rec->mutex);

			if (rec->stop)
				goto out_unlock;

		} while (slist_empty(&rec->slist));

		lightrec_compile_list(rec, thd);
	}

out_unlock:
	pthread_mutex_unlock(&rec->mutex);
	return NULL;
}

struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
{
	struct recompiler *rec;
	unsigned int i, nb_recs, nb_cpus;
	int ret;

	nb_cpus = get_processors_count();
	nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;

	rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
			      + nb_recs * sizeof(*rec->thds));
	if (!rec) {
		pr_err("Cannot create recompiler: Out of memory\n");
		return NULL;
	}

	for (i = 0; i < nb_recs; i++) {
		rec->thds[i].tid = i;
		rec->thds[i].cstate = NULL;
	}

	for (i = 0; i < nb_recs; i++) {
		rec->thds[i].cstate = lightrec_create_cstate(state);
		if (!rec->thds[i].cstate) {
			pr_err("Cannot create recompiler: Out of memory\n");
			goto err_free_cstates;
		}
	}

	rec->state = state;
	rec->stop = false;
	rec->must_flush = false;
	rec->nb_recs = nb_recs;
	slist_init(&rec->slist);

	ret = pthread_cond_init(&rec->cond, NULL);
	if (ret) {
		pr_err("Cannot init cond variable: %d\n", ret);
		goto err_free_cstates;
	}

	ret = pthread_cond_init(&rec->cond2, NULL);
	if (ret) {
		pr_err("Cannot init cond variable: %d\n", ret);
		goto err_cnd_destroy;
	}

	ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
	if (ret) {
		pr_err("Cannot init alloc mutex variable: %d\n", ret);
		goto err_cnd2_destroy;
	}

	ret = pthread_mutex_init(&rec->mutex, NULL);
	if (ret) {
		pr_err("Cannot init mutex variable: %d\n", ret);
		goto err_alloc_mtx_destroy;
	}

	for (i = 0; i < nb_recs; i++) {
		ret = pthread_create(&rec->thds[i].thd, NULL,
				     lightrec_recompiler_thd, &rec->thds[i]);
		if (ret) {
			pr_err("Cannot create recompiler thread: %d\n", ret);
			/* TODO: Handle cleanup properly */
			goto err_mtx_destroy;
		}
	}

	pr_info("Threaded recompiler started with %u workers.\n", nb_recs);

	return rec;

err_mtx_destroy:
	pthread_mutex_destroy(&rec->mutex);
err_alloc_mtx_destroy:
	pthread_mutex_destroy(&rec->alloc_mutex);
err_cnd2_destroy:
	pthread_cond_destroy(&rec->cond2);
err_cnd_destroy:
	pthread_cond_destroy(&rec->cond);
err_free_cstates:
	for (i = 0; i < nb_recs; i++) {
		if (rec->thds[i].cstate)
			lightrec_free_cstate(rec->thds[i].cstate);
	}
	lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	return NULL;
}

void lightrec_free_recompiler(struct recompiler *rec)
{
	unsigned int i;

	rec->stop = true;

	/* Stop the thread */
	pthread_mutex_lock(&rec->mutex);
	pthread_cond_broadcast(&rec->cond);
	lightrec_cancel_list(rec);
	pthread_mutex_unlock(&rec->mutex);

	for (i = 0; i < rec->nb_recs; i++)
		pthread_join(rec->thds[i].thd, NULL);

	for (i = 0; i < rec->nb_recs; i++)
		lightrec_free_cstate(rec->thds[i].cstate);

	pthread_mutex_destroy(&rec->mutex);
	pthread_mutex_destroy(&rec->alloc_mutex);
	pthread_cond_destroy(&rec->cond);
	pthread_cond_destroy(&rec->cond2);
	lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
}

int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
{
	struct slist_elm *elm, *prev;
	struct block_rec *block_rec;
	int ret = 0;

	pthread_mutex_lock(&rec->mutex);

	/* If the recompiler must flush the code cache, we can't add the new
	 * job. It will be re-added next time the block's address is jumped to
	 * again. */
	if (rec->must_flush)
		goto out_unlock;

	/* If the block is marked as dead, don't compile it, it will be removed
	 * as soon as it's safe. */
	if (block->flags & BLOCK_IS_DEAD)
		goto out_unlock;

	for (elm = slist_first(&rec->slist), prev = NULL; elm;
	     prev = elm, elm = elm->next) {
		block_rec = container_of(elm, struct block_rec, slist);

		if (block_rec->block == block) {
			/* The block to compile is already in the queue - bump
			 * it to the top of the list, unless the block is being
			 * recompiled. */
			if (prev && !block_rec->compiling &&
			    !(block->flags & BLOCK_SHOULD_RECOMPILE)) {
				slist_remove_next(prev);
				slist_append(&rec->slist, elm);
			}

			goto out_unlock;
		}
	}

	/* By the time this function was called, the block has been recompiled
	 * and ins't in the wait list anymore. Just return here. */
	if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
		goto out_unlock;

	block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
				    sizeof(*block_rec));
	if (!block_rec) {
		ret = -ENOMEM;
		goto out_unlock;
	}

	pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);

	block_rec->block = block;
	block_rec->compiling = false;

	elm = &rec->slist;

	/* If the block is being recompiled, push it to the end of the queue;
	 * otherwise push it to the front of the queue. */
	if (block->flags & BLOCK_SHOULD_RECOMPILE)
		for (; elm->next; elm = elm->next);

	slist_append(elm, &block_rec->slist);

	/* Signal the thread */
	pthread_cond_signal(&rec->cond);

out_unlock:
	pthread_mutex_unlock(&rec->mutex);

	return ret;
}

void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
{
	struct block_rec *block_rec;
	struct slist_elm *elm;

	pthread_mutex_lock(&rec->mutex);

	while (true) {
		for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
			block_rec = container_of(elm, struct block_rec, slist);

			if (block_rec->block == block) {
				if (lightrec_cancel_block_rec(rec, block_rec))
					goto out_unlock;

				break;
			}
		}

		if (!elm)
			break;
	}

out_unlock:
	pthread_mutex_unlock(&rec->mutex);
}

void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
					  struct block *block, u32 *pc)
{
	bool freed;

	/* There's no point in running the first pass if the block will never
	 * be compiled. Let the main loop run the interpreter instead. */
	if (block->flags & BLOCK_NEVER_COMPILE)
		return NULL;

	/* If the block is already fully tagged, there is no point in running
	 * the first pass. Request a recompilation of the block, and maybe the
	 * interpreter will run the block in the meantime. */
	if (block->flags & BLOCK_FULLY_TAGGED)
		lightrec_recompiler_add(state->rec, block);

	if (likely(block->function)) {
		if (block->flags & BLOCK_FULLY_TAGGED) {
			freed = atomic_flag_test_and_set(&block->op_list_freed);

			if (!freed) {
				pr_debug("Block PC 0x%08x is fully tagged"
					 " - free opcode list\n", block->pc);

				/* The block was already compiled but the opcode list
				 * didn't get freed yet - do it now */
				lightrec_free_opcode_list(state, block);
				block->opcode_list = NULL;
			}
		}

		return block->function;
	}

	/* Mark the opcode list as freed, so that the threaded compiler won't
	 * free it while we're using it in the interpreter. */
	freed = atomic_flag_test_and_set(&block->op_list_freed);

	/* Block wasn't compiled yet - run the interpreter */
	*pc = lightrec_emulate_block(state, block, *pc);

	if (!freed)
		atomic_flag_clear(&block->op_list_freed);

	/* The block got compiled while the interpreter was running.
	 * We can free the opcode list now. */
	if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
	    !atomic_flag_test_and_set(&block->op_list_freed)) {
		pr_debug("Block PC 0x%08x is fully tagged"
			 " - free opcode list\n", block->pc);

		lightrec_free_opcode_list(state, block);
		block->opcode_list = NULL;
	}

	return NULL;
}

void lightrec_code_alloc_lock(struct lightrec_state *state)
{
	pthread_mutex_lock(&state->rec->alloc_mutex);
}

void lightrec_code_alloc_unlock(struct lightrec_state *state)
{
	pthread_mutex_unlock(&state->rec->alloc_mutex);
}
Commit	Line	Data
98fa08a5	1	// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8	2	/*
98fa08a5	3	* Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8 PC	4	*/
d16005f8 PC	5
d8b04acd	6	#include "blockcache.h"
d16005f8 PC	7	#include "debug.h"
	8	#include "interpreter.h"
	9	#include "lightrec-private.h"
	10	#include "memmanager.h"
d8b04acd	11	#include "reaper.h"
a59e5536	12	#include "slist.h"
d16005f8 PC	13
	14	#include <errno.h>
	15	#include <stdatomic.h>
	16	#include <stdbool.h>
	17	#include <stdlib.h>
	18	#include <pthread.h>
98fa08a5 PC	19	#ifdef __linux__
	20	#include <unistd.h>
	21	#endif
d16005f8 PC	22
	23	struct block_rec {
	24	struct block *block;
a59e5536	25	struct slist_elm slist;
98fa08a5 PC	26	bool compiling;
	27	};
	28
	29	struct recompiler_thd {
	30	struct lightrec_cstate *cstate;
	31	unsigned int tid;
	32	pthread_t thd;
d16005f8 PC	33	};
	34
	35	struct recompiler {
	36	struct lightrec_state *state;
d16005f8	37	pthread_cond_t cond;
98fa08a5	38	pthread_cond_t cond2;
d16005f8	39	pthread_mutex_t mutex;
d8b04acd	40	bool stop, must_flush;
a59e5536	41	struct slist_elm slist;
98fa08a5	42
d8b04acd PC	43	pthread_mutex_t alloc_mutex;
d8b04acd PC	44
98fa08a5 PC	45	unsigned int nb_recs;
98fa08a5 PC	46	struct recompiler_thd thds[];
d16005f8 PC	47	};
d16005f8 PC	48
98fa08a5 PC	49	static unsigned int get_processors_count(void)
	50	{
	51	unsigned int nb;
	52
	53	#if defined(PTW32_VERSION)
	54	nb = pthread_num_processors_np();
	55	#elif defined(__APPLE__) \|\| defined(__FreeBSD__)
	56	int count;
	57	size_t size = sizeof(count);
	58
	59	nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
	60	#elif defined(__linux__)
	61	nb = sysconf(_SC_NPROCESSORS_ONLN);
	62	#endif
	63
	64	return nb < 1 ? 1 : nb;
	65	}
	66
	67	static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
	68	{
	69	struct block_rec *block_rec;
	70	struct slist_elm *elm;
	71
	72	for (elm = slist_first(head); elm; elm = elm->next) {
	73	block_rec = container_of(elm, struct block_rec, slist);
	74
	75	if (!block_rec->compiling)
	76	return elm;
	77	}
	78
	79	return NULL;
	80	}
	81
d8b04acd PC	82	static bool lightrec_cancel_block_rec(struct recompiler *rec,
	83	struct block_rec *block_rec)
	84	{
	85	if (block_rec->compiling) {
	86	/* Block is being recompiled - wait for
	87	* completion */
	88	pthread_cond_wait(&rec->cond2, &rec->mutex);
	89
	90	/* We can't guarantee the signal was for us.
	91	* Since block_rec may have been removed while
	92	* we were waiting on the condition, we cannot
	93	* check block_rec->compiling again. The best
	94	* thing is just to restart the function. */
	95	return false;
	96	}
	97
	98	/* Block is not yet being processed - remove it from the list */
	99	slist_remove(&rec->slist, &block_rec->slist);
	100	lightrec_free(rec->state, MEM_FOR_LIGHTREC,
	101	sizeof(*block_rec), block_rec);
	102
	103	return true;
	104	}
	105
	106	static void lightrec_cancel_list(struct recompiler *rec)
	107	{
	108	struct block_rec *block_rec;
	109	struct slist_elm *next;
	110
	111	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
	112	block_rec = container_of(next, struct block_rec, slist);
	113
	114	lightrec_cancel_block_rec(rec, block_rec);
	115	}
	116
	117	pthread_cond_broadcast(&rec->cond2);
	118	}
	119
	120	static void lightrec_flush_code_buffer(struct lightrec_state state, void d)
	121	{
	122	struct recompiler *rec = d;
	123
	124	pthread_mutex_lock(&rec->mutex);
	125
	126	if (rec->must_flush) {
	127	lightrec_remove_outdated_blocks(state->block_cache, NULL);
	128	rec->must_flush = false;
	129	}
	130
	131	pthread_mutex_unlock(&rec->mutex);
	132	}
	133
98fa08a5 PC	134	static void lightrec_compile_list(struct recompiler *rec,
98fa08a5 PC	135	struct recompiler_thd *thd)
d16005f8	136	{
a59e5536	137	struct block_rec *block_rec;
a59e5536	138	struct slist_elm *next;
d16005f8 PC	139	struct block *block;
	140	int ret;
	141
98fa08a5	142	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
a59e5536	143	block_rec = container_of(next, struct block_rec, slist);
98fa08a5	144	block_rec->compiling = true;
a59e5536	145	block = block_rec->block;
d16005f8 PC	146
	147	pthread_mutex_unlock(&rec->mutex);
	148
98fa08a5 PC	149	if (likely(!(block->flags & BLOCK_IS_DEAD))) {
98fa08a5 PC	150	ret = lightrec_compile_block(thd->cstate, block);
d8b04acd PC	151	if (ret == -ENOMEM) {
	152	/* Code buffer is full. Request the reaper to
	153	* flush it. */
	154
	155	pthread_mutex_lock(&rec->mutex);
	156	if (!rec->must_flush) {
	157	lightrec_reaper_add(rec->state->reaper,
	158	lightrec_flush_code_buffer,
	159	rec);
	160	lightrec_cancel_list(rec);
	161	rec->must_flush = true;
	162	}
	163	return;
	164	}
	165
98fa08a5 PC	166	if (ret) {
	167	pr_err("Unable to compile block at PC 0x%x: %d\n",
	168	block->pc, ret);
	169	}
d16005f8 PC	170	}
	171
	172	pthread_mutex_lock(&rec->mutex);
	173
a59e5536	174	slist_remove(&rec->slist, next);
d16005f8	175	lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536	176	sizeof(*block_rec), block_rec);
98fa08a5	177	pthread_cond_signal(&rec->cond2);
d16005f8	178	}
d16005f8 PC	179	}
	180
	181	static void * lightrec_recompiler_thd(void *d)
	182	{
98fa08a5 PC	183	struct recompiler_thd *thd = d;
98fa08a5 PC	184	struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8 PC	185
	186	pthread_mutex_lock(&rec->mutex);
	187
ea17432f	188	while (!rec->stop) {
d16005f8 PC	189	do {
	190	pthread_cond_wait(&rec->cond, &rec->mutex);
	191
a59e5536	192	if (rec->stop)
a59e5536	193	goto out_unlock;
d16005f8	194
a59e5536	195	} while (slist_empty(&rec->slist));
d16005f8	196
98fa08a5	197	lightrec_compile_list(rec, thd);
ea17432f	198	}
a59e5536	199
	200	out_unlock:
	201	pthread_mutex_unlock(&rec->mutex);
	202	return NULL;
d16005f8 PC	203	}
	204
	205	struct recompiler lightrec_recompiler_init(struct lightrec_state state)
	206	{
	207	struct recompiler *rec;
98fa08a5	208	unsigned int i, nb_recs, nb_cpus;
d16005f8 PC	209	int ret;
d16005f8 PC	210
98fa08a5 PC	211	nb_cpus = get_processors_count();
	212	nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
	213
	214	rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
	215	+ nb_recs * sizeof(*rec->thds));
d16005f8 PC	216	if (!rec) {
	217	pr_err("Cannot create recompiler: Out of memory\n");
	218	return NULL;
	219	}
	220
98fa08a5 PC	221	for (i = 0; i < nb_recs; i++) {
	222	rec->thds[i].tid = i;
	223	rec->thds[i].cstate = NULL;
	224	}
	225
	226	for (i = 0; i < nb_recs; i++) {
	227	rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e	228	if (!rec->thds[i].cstate) {
98fa08a5 PC	229	pr_err("Cannot create recompiler: Out of memory\n");
	230	goto err_free_cstates;
	231	}
	232	}
	233
d16005f8 PC	234	rec->state = state;
d16005f8 PC	235	rec->stop = false;
d8b04acd	236	rec->must_flush = false;
98fa08a5	237	rec->nb_recs = nb_recs;
a59e5536	238	slist_init(&rec->slist);
d16005f8 PC	239
	240	ret = pthread_cond_init(&rec->cond, NULL);
	241	if (ret) {
	242	pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5	243	goto err_free_cstates;
d16005f8 PC	244	}
d16005f8 PC	245
98fa08a5	246	ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8	247	if (ret) {
98fa08a5	248	pr_err("Cannot init cond variable: %d\n", ret);
d16005f8 PC	249	goto err_cnd_destroy;
	250	}
	251
d8b04acd PC	252	ret = pthread_mutex_init(&rec->alloc_mutex, NULL);
	253	if (ret) {
	254	pr_err("Cannot init alloc mutex variable: %d\n", ret);
	255	goto err_cnd2_destroy;
	256	}
	257
98fa08a5	258	ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8	259	if (ret) {
98fa08a5	260	pr_err("Cannot init mutex variable: %d\n", ret);
d8b04acd	261	goto err_alloc_mtx_destroy;
d16005f8 PC	262	}
d16005f8 PC	263
98fa08a5 PC	264	for (i = 0; i < nb_recs; i++) {
	265	ret = pthread_create(&rec->thds[i].thd, NULL,
	266	lightrec_recompiler_thd, &rec->thds[i]);
	267	if (ret) {
	268	pr_err("Cannot create recompiler thread: %d\n", ret);
	269	/* TODO: Handle cleanup properly */
	270	goto err_mtx_destroy;
	271	}
	272	}
	273
	274	pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
	275
d16005f8 PC	276	return rec;
	277
	278	err_mtx_destroy:
	279	pthread_mutex_destroy(&rec->mutex);
d8b04acd PC	280	err_alloc_mtx_destroy:
d8b04acd PC	281	pthread_mutex_destroy(&rec->alloc_mutex);
98fa08a5 PC	282	err_cnd2_destroy:
98fa08a5 PC	283	pthread_cond_destroy(&rec->cond2);
d16005f8 PC	284	err_cnd_destroy:
d16005f8 PC	285	pthread_cond_destroy(&rec->cond);
98fa08a5 PC	286	err_free_cstates:
	287	for (i = 0; i < nb_recs; i++) {
	288	if (rec->thds[i].cstate)
	289	lightrec_free_cstate(rec->thds[i].cstate);
	290	}
d16005f8 PC	291	lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	292	return NULL;
	293	}
	294
	295	void lightrec_free_recompiler(struct recompiler *rec)
	296	{
98fa08a5 PC	297	unsigned int i;
98fa08a5 PC	298
d16005f8 PC	299	rec->stop = true;
	300
	301	/* Stop the thread */
	302	pthread_mutex_lock(&rec->mutex);
98fa08a5	303	pthread_cond_broadcast(&rec->cond);
d8b04acd	304	lightrec_cancel_list(rec);
d16005f8	305	pthread_mutex_unlock(&rec->mutex);
98fa08a5 PC	306
	307	for (i = 0; i < rec->nb_recs; i++)
	308	pthread_join(rec->thds[i].thd, NULL);
	309
	310	for (i = 0; i < rec->nb_recs; i++)
	311	lightrec_free_cstate(rec->thds[i].cstate);
d16005f8 PC	312
d16005f8 PC	313	pthread_mutex_destroy(&rec->mutex);
d8b04acd	314	pthread_mutex_destroy(&rec->alloc_mutex);
d16005f8	315	pthread_cond_destroy(&rec->cond);
98fa08a5	316	pthread_cond_destroy(&rec->cond2);
d16005f8 PC	317	lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	318	}
	319
	320	int lightrec_recompiler_add(struct recompiler rec, struct block block)
	321	{
a59e5536	322	struct slist_elm elm, prev;
	323	struct block_rec *block_rec;
	324	int ret = 0;
d16005f8 PC	325
	326	pthread_mutex_lock(&rec->mutex);
	327
d8b04acd PC	328	/* If the recompiler must flush the code cache, we can't add the new
	329	* job. It will be re-added next time the block's address is jumped to
	330	* again. */
	331	if (rec->must_flush)
	332	goto out_unlock;
	333
a59e5536	334	/* If the block is marked as dead, don't compile it, it will be removed
	335	* as soon as it's safe. */
	336	if (block->flags & BLOCK_IS_DEAD)
	337	goto out_unlock;
	338
	339	for (elm = slist_first(&rec->slist), prev = NULL; elm;
	340	prev = elm, elm = elm->next) {
	341	block_rec = container_of(elm, struct block_rec, slist);
	342
d16005f8 PC	343	if (block_rec->block == block) {
d16005f8 PC	344	/* The block to compile is already in the queue - bump
a59e5536	345	* it to the top of the list, unless the block is being
a59e5536	346	* recompiled. */
98fa08a5 PC	347	if (prev && !block_rec->compiling &&
98fa08a5 PC	348	!(block->flags & BLOCK_SHOULD_RECOMPILE)) {
a59e5536	349	slist_remove_next(prev);
a59e5536	350	slist_append(&rec->slist, elm);
d16005f8 PC	351	}
d16005f8 PC	352
a59e5536	353	goto out_unlock;
d16005f8 PC	354	}
	355	}
	356
	357	/* By the time this function was called, the block has been recompiled
	358	* and ins't in the wait list anymore. Just return here. */
a59e5536	359	if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
a59e5536	360	goto out_unlock;
d16005f8 PC	361
	362	block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
	363	sizeof(*block_rec));
	364	if (!block_rec) {
a59e5536	365	ret = -ENOMEM;
a59e5536	366	goto out_unlock;
d16005f8 PC	367	}
	368
	369	pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);
	370
	371	block_rec->block = block;
98fa08a5	372	block_rec->compiling = false;
a59e5536	373
	374	elm = &rec->slist;
	375
	376	/* If the block is being recompiled, push it to the end of the queue;
	377	* otherwise push it to the front of the queue. */
	378	if (block->flags & BLOCK_SHOULD_RECOMPILE)
	379	for (; elm->next; elm = elm->next);
	380
	381	slist_append(elm, &block_rec->slist);
d16005f8 PC	382
	383	/* Signal the thread */
	384	pthread_cond_signal(&rec->cond);
d16005f8	385
a59e5536	386	out_unlock:
a59e5536	387	pthread_mutex_unlock(&rec->mutex);
98fa08a5	388
a59e5536	389	return ret;
d16005f8 PC	390	}
	391
	392	void lightrec_recompiler_remove(struct recompiler rec, struct block block)
	393	{
	394	struct block_rec *block_rec;
a59e5536	395	struct slist_elm *elm;
d16005f8 PC	396
	397	pthread_mutex_lock(&rec->mutex);
	398
98fa08a5 PC	399	while (true) {
	400	for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
	401	block_rec = container_of(elm, struct block_rec, slist);
a59e5536	402
d8b04acd PC	403	if (block_rec->block == block) {
	404	if (lightrec_cancel_block_rec(rec, block_rec))
	405	goto out_unlock;
98fa08a5	406
98fa08a5	407	break;
d16005f8	408	}
98fa08a5	409	}
d16005f8	410
98fa08a5	411	if (!elm)
d16005f8	412	break;
d16005f8 PC	413	}
d16005f8 PC	414
98fa08a5	415	out_unlock:
d16005f8 PC	416	pthread_mutex_unlock(&rec->mutex);
	417	}
	418
98fa08a5 PC	419	void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
98fa08a5 PC	420	struct block block, u32 pc)
d16005f8 PC	421	{
	422	bool freed;
	423
98fa08a5 PC	424	/* There's no point in running the first pass if the block will never
	425	* be compiled. Let the main loop run the interpreter instead. */
	426	if (block->flags & BLOCK_NEVER_COMPILE)
	427	return NULL;
	428
	429	/* If the block is already fully tagged, there is no point in running
	430	* the first pass. Request a recompilation of the block, and maybe the
	431	* interpreter will run the block in the meantime. */
	432	if (block->flags & BLOCK_FULLY_TAGGED)
	433	lightrec_recompiler_add(state->rec, block);
	434
d16005f8 PC	435	if (likely(block->function)) {
	436	if (block->flags & BLOCK_FULLY_TAGGED) {
	437	freed = atomic_flag_test_and_set(&block->op_list_freed);
	438
	439	if (!freed) {
	440	pr_debug("Block PC 0x%08x is fully tagged"
	441	" - free opcode list\n", block->pc);
	442
	443	/* The block was already compiled but the opcode list
	444	* didn't get freed yet - do it now */
98fa08a5	445	lightrec_free_opcode_list(state, block);
d16005f8 PC	446	block->opcode_list = NULL;
	447	}
	448	}
	449
	450	return block->function;
	451	}
	452
	453	/* Mark the opcode list as freed, so that the threaded compiler won't
	454	* free it while we're using it in the interpreter. */
	455	freed = atomic_flag_test_and_set(&block->op_list_freed);
	456
	457	/* Block wasn't compiled yet - run the interpreter */
98fa08a5	458	pc = lightrec_emulate_block(state, block, pc);
d16005f8 PC	459
	460	if (!freed)
	461	atomic_flag_clear(&block->op_list_freed);
	462
	463	/* The block got compiled while the interpreter was running.
	464	* We can free the opcode list now. */
	465	if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
	466	!atomic_flag_test_and_set(&block->op_list_freed)) {
	467	pr_debug("Block PC 0x%08x is fully tagged"
	468	" - free opcode list\n", block->pc);
	469
98fa08a5	470	lightrec_free_opcode_list(state, block);
d16005f8 PC	471	block->opcode_list = NULL;
	472	}
	473
	474	return NULL;
	475	}
d8b04acd PC	476
	477	void lightrec_code_alloc_lock(struct lightrec_state *state)
	478	{
	479	pthread_mutex_lock(&state->rec->alloc_mutex);
	480	}
	481
	482	void lightrec_code_alloc_unlock(struct lightrec_state *state)
	483	{
	484	pthread_mutex_unlock(&state->rec->alloc_mutex);
	485	}