[pcsx_rearmed.git] / deps / lightrec / recompiler.c

// SPDX-License-Identifier: LGPL-2.1-or-later
/*
 * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
 */

#include "debug.h"
#include "interpreter.h"
#include "lightrec-private.h"
#include "memmanager.h"
#include "slist.h"

#include <errno.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdlib.h>
#include <pthread.h>
#ifdef __linux__
#include <unistd.h>
#endif

struct block_rec {
	struct block *block;
	struct slist_elm slist;
	bool compiling;
};

struct recompiler_thd {
	struct lightrec_cstate *cstate;
	unsigned int tid;
	pthread_t thd;
};

struct recompiler {
	struct lightrec_state *state;
	pthread_cond_t cond;
	pthread_cond_t cond2;
	pthread_mutex_t mutex;
	bool stop;
	struct slist_elm slist;

	unsigned int nb_recs;
	struct recompiler_thd thds[];
};

static unsigned int get_processors_count(void)
{
	unsigned int nb;

#if defined(PTW32_VERSION)
        nb = pthread_num_processors_np();
#elif defined(__APPLE__) || defined(__FreeBSD__)
        int count;
        size_t size = sizeof(count);

        nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
#elif defined(__linux__)
	nb = sysconf(_SC_NPROCESSORS_ONLN);
#endif

	return nb < 1 ? 1 : nb;
}

static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
{
	struct block_rec *block_rec;
	struct slist_elm *elm;

	for (elm = slist_first(head); elm; elm = elm->next) {
		block_rec = container_of(elm, struct block_rec, slist);

		if (!block_rec->compiling)
			return elm;
	}

	return NULL;
}

static void lightrec_compile_list(struct recompiler *rec,
				  struct recompiler_thd *thd)
{
	struct block_rec *block_rec;
	struct slist_elm *next;
	struct block *block;
	int ret;

	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
		block_rec = container_of(next, struct block_rec, slist);
		block_rec->compiling = true;
		block = block_rec->block;

		pthread_mutex_unlock(&rec->mutex);

		if (likely(!(block->flags & BLOCK_IS_DEAD))) {
			ret = lightrec_compile_block(thd->cstate, block);
			if (ret) {
				pr_err("Unable to compile block at PC 0x%x: %d\n",
				       block->pc, ret);
			}
		}

		pthread_mutex_lock(&rec->mutex);

		slist_remove(&rec->slist, next);
		lightrec_free(rec->state, MEM_FOR_LIGHTREC,
			      sizeof(*block_rec), block_rec);
		pthread_cond_signal(&rec->cond2);
	}
}

static void * lightrec_recompiler_thd(void *d)
{
	struct recompiler_thd *thd = d;
	struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);

	pthread_mutex_lock(&rec->mutex);

	while (!rec->stop) {
		do {
			pthread_cond_wait(&rec->cond, &rec->mutex);

			if (rec->stop)
				goto out_unlock;

		} while (slist_empty(&rec->slist));

		lightrec_compile_list(rec, thd);
	}

out_unlock:
	pthread_mutex_unlock(&rec->mutex);
	return NULL;
}

struct recompiler *lightrec_recompiler_init(struct lightrec_state *state)
{
	struct recompiler *rec;
	unsigned int i, nb_recs, nb_cpus;
	int ret;

	nb_cpus = get_processors_count();
	nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;

	rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
			      + nb_recs * sizeof(*rec->thds));
	if (!rec) {
		pr_err("Cannot create recompiler: Out of memory\n");
		return NULL;
	}

	for (i = 0; i < nb_recs; i++) {
		rec->thds[i].tid = i;
		rec->thds[i].cstate = NULL;
	}

	for (i = 0; i < nb_recs; i++) {
		rec->thds[i].cstate = lightrec_create_cstate(state);
		if (!rec->thds[i].cstate) {
			pr_err("Cannot create recompiler: Out of memory\n");
			goto err_free_cstates;
		}
	}

	rec->state = state;
	rec->stop = false;
	rec->nb_recs = nb_recs;
	slist_init(&rec->slist);

	ret = pthread_cond_init(&rec->cond, NULL);
	if (ret) {
		pr_err("Cannot init cond variable: %d\n", ret);
		goto err_free_cstates;
	}

	ret = pthread_cond_init(&rec->cond2, NULL);
	if (ret) {
		pr_err("Cannot init cond variable: %d\n", ret);
		goto err_cnd_destroy;
	}

	ret = pthread_mutex_init(&rec->mutex, NULL);
	if (ret) {
		pr_err("Cannot init mutex variable: %d\n", ret);
		goto err_cnd2_destroy;
	}

	for (i = 0; i < nb_recs; i++) {
		ret = pthread_create(&rec->thds[i].thd, NULL,
				     lightrec_recompiler_thd, &rec->thds[i]);
		if (ret) {
			pr_err("Cannot create recompiler thread: %d\n", ret);
			/* TODO: Handle cleanup properly */
			goto err_mtx_destroy;
		}
	}

	pr_info("Threaded recompiler started with %u workers.\n", nb_recs);

	return rec;

err_mtx_destroy:
	pthread_mutex_destroy(&rec->mutex);
err_cnd2_destroy:
	pthread_cond_destroy(&rec->cond2);
err_cnd_destroy:
	pthread_cond_destroy(&rec->cond);
err_free_cstates:
	for (i = 0; i < nb_recs; i++) {
		if (rec->thds[i].cstate)
			lightrec_free_cstate(rec->thds[i].cstate);
	}
	lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	return NULL;
}

void lightrec_free_recompiler(struct recompiler *rec)
{
	unsigned int i;

	rec->stop = true;

	/* Stop the thread */
	pthread_mutex_lock(&rec->mutex);
	pthread_cond_broadcast(&rec->cond);
	pthread_mutex_unlock(&rec->mutex);

	for (i = 0; i < rec->nb_recs; i++)
		pthread_join(rec->thds[i].thd, NULL);

	for (i = 0; i < rec->nb_recs; i++)
		lightrec_free_cstate(rec->thds[i].cstate);

	pthread_mutex_destroy(&rec->mutex);
	pthread_cond_destroy(&rec->cond);
	pthread_cond_destroy(&rec->cond2);
	lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
}

int lightrec_recompiler_add(struct recompiler *rec, struct block *block)
{
	struct slist_elm *elm, *prev;
	struct block_rec *block_rec;
	int ret = 0;

	pthread_mutex_lock(&rec->mutex);

	/* If the block is marked as dead, don't compile it, it will be removed
	 * as soon as it's safe. */
	if (block->flags & BLOCK_IS_DEAD)
		goto out_unlock;

	for (elm = slist_first(&rec->slist), prev = NULL; elm;
	     prev = elm, elm = elm->next) {
		block_rec = container_of(elm, struct block_rec, slist);

		if (block_rec->block == block) {
			/* The block to compile is already in the queue - bump
			 * it to the top of the list, unless the block is being
			 * recompiled. */
			if (prev && !block_rec->compiling &&
			    !(block->flags & BLOCK_SHOULD_RECOMPILE)) {
				slist_remove_next(prev);
				slist_append(&rec->slist, elm);
			}

			goto out_unlock;
		}
	}

	/* By the time this function was called, the block has been recompiled
	 * and ins't in the wait list anymore. Just return here. */
	if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
		goto out_unlock;

	block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
				    sizeof(*block_rec));
	if (!block_rec) {
		ret = -ENOMEM;
		goto out_unlock;
	}

	pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);

	block_rec->block = block;
	block_rec->compiling = false;

	elm = &rec->slist;

	/* If the block is being recompiled, push it to the end of the queue;
	 * otherwise push it to the front of the queue. */
	if (block->flags & BLOCK_SHOULD_RECOMPILE)
		for (; elm->next; elm = elm->next);

	slist_append(elm, &block_rec->slist);

	/* Signal the thread */
	pthread_cond_signal(&rec->cond);

out_unlock:
	pthread_mutex_unlock(&rec->mutex);

	return ret;
}

void lightrec_recompiler_remove(struct recompiler *rec, struct block *block)
{
	struct block_rec *block_rec;
	struct slist_elm *elm;

	pthread_mutex_lock(&rec->mutex);

	while (true) {
		for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
			block_rec = container_of(elm, struct block_rec, slist);

			if (block_rec->block != block)
				continue;

			if (block_rec->compiling) {
				/* Block is being recompiled - wait for
				 * completion */
				pthread_cond_wait(&rec->cond2, &rec->mutex);

				/* We can't guarantee the signal was for us.
				 * Since block_rec may have been removed while
				 * we were waiting on the condition, we cannot
				 * check block_rec->compiling again. The best
				 * thing is just to restart the function. */
				break;
			} else {
				/* Block is not yet being processed - remove it
				 * from the list */
				slist_remove(&rec->slist, elm);
				lightrec_free(rec->state, MEM_FOR_LIGHTREC,
					      sizeof(*block_rec), block_rec);

				goto out_unlock;
			}
		}

		if (!elm)
			break;
	}

out_unlock:
	pthread_mutex_unlock(&rec->mutex);
}

void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
					  struct block *block, u32 *pc)
{
	bool freed;

	/* There's no point in running the first pass if the block will never
	 * be compiled. Let the main loop run the interpreter instead. */
	if (block->flags & BLOCK_NEVER_COMPILE)
		return NULL;

	/* If the block is already fully tagged, there is no point in running
	 * the first pass. Request a recompilation of the block, and maybe the
	 * interpreter will run the block in the meantime. */
	if (block->flags & BLOCK_FULLY_TAGGED)
		lightrec_recompiler_add(state->rec, block);

	if (likely(block->function)) {
		if (block->flags & BLOCK_FULLY_TAGGED) {
			freed = atomic_flag_test_and_set(&block->op_list_freed);

			if (!freed) {
				pr_debug("Block PC 0x%08x is fully tagged"
					 " - free opcode list\n", block->pc);

				/* The block was already compiled but the opcode list
				 * didn't get freed yet - do it now */
				lightrec_free_opcode_list(state, block);
				block->opcode_list = NULL;
			}
		}

		return block->function;
	}

	/* Mark the opcode list as freed, so that the threaded compiler won't
	 * free it while we're using it in the interpreter. */
	freed = atomic_flag_test_and_set(&block->op_list_freed);

	/* Block wasn't compiled yet - run the interpreter */
	*pc = lightrec_emulate_block(state, block, *pc);

	if (!freed)
		atomic_flag_clear(&block->op_list_freed);

	/* The block got compiled while the interpreter was running.
	 * We can free the opcode list now. */
	if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
	    !atomic_flag_test_and_set(&block->op_list_freed)) {
		pr_debug("Block PC 0x%08x is fully tagged"
			 " - free opcode list\n", block->pc);

		lightrec_free_opcode_list(state, block);
		block->opcode_list = NULL;
	}

	return NULL;
}
Commit	Line	Data
98fa08a5	1	// SPDX-License-Identifier: LGPL-2.1-or-later
d16005f8	2	/*
98fa08a5	3	* Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net>
d16005f8 PC	4	*/
	5
	6	#include "debug.h"
	7	#include "interpreter.h"
	8	#include "lightrec-private.h"
	9	#include "memmanager.h"
a59e5536	10	#include "slist.h"
d16005f8 PC	11
	12	#include <errno.h>
	13	#include <stdatomic.h>
	14	#include <stdbool.h>
	15	#include <stdlib.h>
	16	#include <pthread.h>
98fa08a5 PC	17	#ifdef __linux__
	18	#include <unistd.h>
	19	#endif
d16005f8 PC	20
	21	struct block_rec {
	22	struct block *block;
a59e5536	23	struct slist_elm slist;
98fa08a5 PC	24	bool compiling;
	25	};
	26
	27	struct recompiler_thd {
	28	struct lightrec_cstate *cstate;
	29	unsigned int tid;
	30	pthread_t thd;
d16005f8 PC	31	};
	32
	33	struct recompiler {
	34	struct lightrec_state *state;
d16005f8	35	pthread_cond_t cond;
98fa08a5	36	pthread_cond_t cond2;
d16005f8 PC	37	pthread_mutex_t mutex;
d16005f8 PC	38	bool stop;
a59e5536	39	struct slist_elm slist;
98fa08a5 PC	40
	41	unsigned int nb_recs;
	42	struct recompiler_thd thds[];
d16005f8 PC	43	};
d16005f8 PC	44
98fa08a5 PC	45	static unsigned int get_processors_count(void)
	46	{
	47	unsigned int nb;
	48
	49	#if defined(PTW32_VERSION)
	50	nb = pthread_num_processors_np();
	51	#elif defined(__APPLE__) \|\| defined(__FreeBSD__)
	52	int count;
	53	size_t size = sizeof(count);
	54
	55	nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count;
	56	#elif defined(__linux__)
	57	nb = sysconf(_SC_NPROCESSORS_ONLN);
	58	#endif
	59
	60	return nb < 1 ? 1 : nb;
	61	}
	62
	63	static struct slist_elm * lightrec_get_first_elm(struct slist_elm *head)
	64	{
	65	struct block_rec *block_rec;
	66	struct slist_elm *elm;
	67
	68	for (elm = slist_first(head); elm; elm = elm->next) {
	69	block_rec = container_of(elm, struct block_rec, slist);
	70
	71	if (!block_rec->compiling)
	72	return elm;
	73	}
	74
	75	return NULL;
	76	}
	77
	78	static void lightrec_compile_list(struct recompiler *rec,
	79	struct recompiler_thd *thd)
d16005f8	80	{
a59e5536	81	struct block_rec *block_rec;
a59e5536	82	struct slist_elm *next;
d16005f8 PC	83	struct block *block;
	84	int ret;
	85
98fa08a5	86	while (!!(next = lightrec_get_first_elm(&rec->slist))) {
a59e5536	87	block_rec = container_of(next, struct block_rec, slist);
98fa08a5	88	block_rec->compiling = true;
a59e5536	89	block = block_rec->block;
d16005f8 PC	90
	91	pthread_mutex_unlock(&rec->mutex);
	92
98fa08a5 PC	93	if (likely(!(block->flags & BLOCK_IS_DEAD))) {
	94	ret = lightrec_compile_block(thd->cstate, block);
	95	if (ret) {
	96	pr_err("Unable to compile block at PC 0x%x: %d\n",
	97	block->pc, ret);
	98	}
d16005f8 PC	99	}
	100
	101	pthread_mutex_lock(&rec->mutex);
	102
a59e5536	103	slist_remove(&rec->slist, next);
d16005f8	104	lightrec_free(rec->state, MEM_FOR_LIGHTREC,
a59e5536	105	sizeof(*block_rec), block_rec);
98fa08a5	106	pthread_cond_signal(&rec->cond2);
d16005f8	107	}
d16005f8 PC	108	}
	109
	110	static void * lightrec_recompiler_thd(void *d)
	111	{
98fa08a5 PC	112	struct recompiler_thd *thd = d;
98fa08a5 PC	113	struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]);
d16005f8 PC	114
	115	pthread_mutex_lock(&rec->mutex);
	116
ea17432f	117	while (!rec->stop) {
d16005f8 PC	118	do {
	119	pthread_cond_wait(&rec->cond, &rec->mutex);
	120
a59e5536	121	if (rec->stop)
a59e5536	122	goto out_unlock;
d16005f8	123
a59e5536	124	} while (slist_empty(&rec->slist));
d16005f8	125
98fa08a5	126	lightrec_compile_list(rec, thd);
ea17432f	127	}
a59e5536	128
	129	out_unlock:
	130	pthread_mutex_unlock(&rec->mutex);
	131	return NULL;
d16005f8 PC	132	}
	133
	134	struct recompiler lightrec_recompiler_init(struct lightrec_state state)
	135	{
	136	struct recompiler *rec;
98fa08a5	137	unsigned int i, nb_recs, nb_cpus;
d16005f8 PC	138	int ret;
d16005f8 PC	139
98fa08a5 PC	140	nb_cpus = get_processors_count();
	141	nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1;
	142
	143	rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec)
	144	+ nb_recs * sizeof(*rec->thds));
d16005f8 PC	145	if (!rec) {
	146	pr_err("Cannot create recompiler: Out of memory\n");
	147	return NULL;
	148	}
	149
98fa08a5 PC	150	for (i = 0; i < nb_recs; i++) {
	151	rec->thds[i].tid = i;
	152	rec->thds[i].cstate = NULL;
	153	}
	154
	155	for (i = 0; i < nb_recs; i++) {
	156	rec->thds[i].cstate = lightrec_create_cstate(state);
25851a1e	157	if (!rec->thds[i].cstate) {
98fa08a5 PC	158	pr_err("Cannot create recompiler: Out of memory\n");
	159	goto err_free_cstates;
	160	}
	161	}
	162
d16005f8 PC	163	rec->state = state;
d16005f8 PC	164	rec->stop = false;
98fa08a5	165	rec->nb_recs = nb_recs;
a59e5536	166	slist_init(&rec->slist);
d16005f8 PC	167
	168	ret = pthread_cond_init(&rec->cond, NULL);
	169	if (ret) {
	170	pr_err("Cannot init cond variable: %d\n", ret);
98fa08a5	171	goto err_free_cstates;
d16005f8 PC	172	}
d16005f8 PC	173
98fa08a5	174	ret = pthread_cond_init(&rec->cond2, NULL);
d16005f8	175	if (ret) {
98fa08a5	176	pr_err("Cannot init cond variable: %d\n", ret);
d16005f8 PC	177	goto err_cnd_destroy;
	178	}
	179
98fa08a5	180	ret = pthread_mutex_init(&rec->mutex, NULL);
d16005f8	181	if (ret) {
98fa08a5 PC	182	pr_err("Cannot init mutex variable: %d\n", ret);
98fa08a5 PC	183	goto err_cnd2_destroy;
d16005f8 PC	184	}
d16005f8 PC	185
98fa08a5 PC	186	for (i = 0; i < nb_recs; i++) {
	187	ret = pthread_create(&rec->thds[i].thd, NULL,
	188	lightrec_recompiler_thd, &rec->thds[i]);
	189	if (ret) {
	190	pr_err("Cannot create recompiler thread: %d\n", ret);
	191	/* TODO: Handle cleanup properly */
	192	goto err_mtx_destroy;
	193	}
	194	}
	195
	196	pr_info("Threaded recompiler started with %u workers.\n", nb_recs);
	197
d16005f8 PC	198	return rec;
	199
	200	err_mtx_destroy:
	201	pthread_mutex_destroy(&rec->mutex);
98fa08a5 PC	202	err_cnd2_destroy:
98fa08a5 PC	203	pthread_cond_destroy(&rec->cond2);
d16005f8 PC	204	err_cnd_destroy:
d16005f8 PC	205	pthread_cond_destroy(&rec->cond);
98fa08a5 PC	206	err_free_cstates:
	207	for (i = 0; i < nb_recs; i++) {
	208	if (rec->thds[i].cstate)
	209	lightrec_free_cstate(rec->thds[i].cstate);
	210	}
d16005f8 PC	211	lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	212	return NULL;
	213	}
	214
	215	void lightrec_free_recompiler(struct recompiler *rec)
	216	{
98fa08a5 PC	217	unsigned int i;
98fa08a5 PC	218
d16005f8 PC	219	rec->stop = true;
	220
	221	/* Stop the thread */
	222	pthread_mutex_lock(&rec->mutex);
98fa08a5	223	pthread_cond_broadcast(&rec->cond);
d16005f8	224	pthread_mutex_unlock(&rec->mutex);
98fa08a5 PC	225
	226	for (i = 0; i < rec->nb_recs; i++)
	227	pthread_join(rec->thds[i].thd, NULL);
	228
	229	for (i = 0; i < rec->nb_recs; i++)
	230	lightrec_free_cstate(rec->thds[i].cstate);
d16005f8 PC	231
	232	pthread_mutex_destroy(&rec->mutex);
	233	pthread_cond_destroy(&rec->cond);
98fa08a5	234	pthread_cond_destroy(&rec->cond2);
d16005f8 PC	235	lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec);
	236	}
	237
	238	int lightrec_recompiler_add(struct recompiler rec, struct block block)
	239	{
a59e5536	240	struct slist_elm elm, prev;
	241	struct block_rec *block_rec;
	242	int ret = 0;
d16005f8 PC	243
	244	pthread_mutex_lock(&rec->mutex);
	245
a59e5536	246	/* If the block is marked as dead, don't compile it, it will be removed
	247	* as soon as it's safe. */
	248	if (block->flags & BLOCK_IS_DEAD)
	249	goto out_unlock;
	250
	251	for (elm = slist_first(&rec->slist), prev = NULL; elm;
	252	prev = elm, elm = elm->next) {
	253	block_rec = container_of(elm, struct block_rec, slist);
	254
d16005f8 PC	255	if (block_rec->block == block) {
d16005f8 PC	256	/* The block to compile is already in the queue - bump
a59e5536	257	* it to the top of the list, unless the block is being
a59e5536	258	* recompiled. */
98fa08a5 PC	259	if (prev && !block_rec->compiling &&
98fa08a5 PC	260	!(block->flags & BLOCK_SHOULD_RECOMPILE)) {
a59e5536	261	slist_remove_next(prev);
a59e5536	262	slist_append(&rec->slist, elm);
d16005f8 PC	263	}
d16005f8 PC	264
a59e5536	265	goto out_unlock;
d16005f8 PC	266	}
	267	}
	268
	269	/* By the time this function was called, the block has been recompiled
	270	* and ins't in the wait list anymore. Just return here. */
a59e5536	271	if (block->function && !(block->flags & BLOCK_SHOULD_RECOMPILE))
a59e5536	272	goto out_unlock;
d16005f8 PC	273
	274	block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC,
	275	sizeof(*block_rec));
	276	if (!block_rec) {
a59e5536	277	ret = -ENOMEM;
a59e5536	278	goto out_unlock;
d16005f8 PC	279	}
	280
	281	pr_debug("Adding block PC 0x%x to recompiler\n", block->pc);
	282
	283	block_rec->block = block;
98fa08a5	284	block_rec->compiling = false;
a59e5536	285
	286	elm = &rec->slist;
	287
	288	/* If the block is being recompiled, push it to the end of the queue;
	289	* otherwise push it to the front of the queue. */
	290	if (block->flags & BLOCK_SHOULD_RECOMPILE)
	291	for (; elm->next; elm = elm->next);
	292
	293	slist_append(elm, &block_rec->slist);
d16005f8 PC	294
	295	/* Signal the thread */
	296	pthread_cond_signal(&rec->cond);
d16005f8	297
a59e5536	298	out_unlock:
a59e5536	299	pthread_mutex_unlock(&rec->mutex);
98fa08a5	300
a59e5536	301	return ret;
d16005f8 PC	302	}
	303
	304	void lightrec_recompiler_remove(struct recompiler rec, struct block block)
	305	{
	306	struct block_rec *block_rec;
a59e5536	307	struct slist_elm *elm;
d16005f8 PC	308
	309	pthread_mutex_lock(&rec->mutex);
	310
98fa08a5 PC	311	while (true) {
	312	for (elm = slist_first(&rec->slist); elm; elm = elm->next) {
	313	block_rec = container_of(elm, struct block_rec, slist);
a59e5536	314
98fa08a5 PC	315	if (block_rec->block != block)
	316	continue;
	317
	318	if (block_rec->compiling) {
d16005f8 PC	319	/* Block is being recompiled - wait for
d16005f8 PC	320	* completion */
98fa08a5 PC	321	pthread_cond_wait(&rec->cond2, &rec->mutex);
	322
	323	/* We can't guarantee the signal was for us.
	324	* Since block_rec may have been removed while
	325	* we were waiting on the condition, we cannot
	326	* check block_rec->compiling again. The best
	327	* thing is just to restart the function. */
	328	break;
d16005f8 PC	329	} else {
	330	/* Block is not yet being processed - remove it
	331	* from the list */
a59e5536	332	slist_remove(&rec->slist, elm);
d16005f8 PC	333	lightrec_free(rec->state, MEM_FOR_LIGHTREC,
d16005f8 PC	334	sizeof(*block_rec), block_rec);
98fa08a5 PC	335
98fa08a5 PC	336	goto out_unlock;
d16005f8	337	}
98fa08a5	338	}
d16005f8	339
98fa08a5	340	if (!elm)
d16005f8	341	break;
d16005f8 PC	342	}
d16005f8 PC	343
98fa08a5	344	out_unlock:
d16005f8 PC	345	pthread_mutex_unlock(&rec->mutex);
	346	}
	347
98fa08a5 PC	348	void * lightrec_recompiler_run_first_pass(struct lightrec_state *state,
98fa08a5 PC	349	struct block block, u32 pc)
d16005f8 PC	350	{
	351	bool freed;
	352
98fa08a5 PC	353	/* There's no point in running the first pass if the block will never
	354	* be compiled. Let the main loop run the interpreter instead. */
	355	if (block->flags & BLOCK_NEVER_COMPILE)
	356	return NULL;
	357
	358	/* If the block is already fully tagged, there is no point in running
	359	* the first pass. Request a recompilation of the block, and maybe the
	360	* interpreter will run the block in the meantime. */
	361	if (block->flags & BLOCK_FULLY_TAGGED)
	362	lightrec_recompiler_add(state->rec, block);
	363
d16005f8 PC	364	if (likely(block->function)) {
	365	if (block->flags & BLOCK_FULLY_TAGGED) {
	366	freed = atomic_flag_test_and_set(&block->op_list_freed);
	367
	368	if (!freed) {
	369	pr_debug("Block PC 0x%08x is fully tagged"
	370	" - free opcode list\n", block->pc);
	371
	372	/* The block was already compiled but the opcode list
	373	* didn't get freed yet - do it now */
98fa08a5	374	lightrec_free_opcode_list(state, block);
d16005f8 PC	375	block->opcode_list = NULL;
	376	}
	377	}
	378
	379	return block->function;
	380	}
	381
	382	/* Mark the opcode list as freed, so that the threaded compiler won't
	383	* free it while we're using it in the interpreter. */
	384	freed = atomic_flag_test_and_set(&block->op_list_freed);
	385
	386	/* Block wasn't compiled yet - run the interpreter */
98fa08a5	387	pc = lightrec_emulate_block(state, block, pc);
d16005f8 PC	388
	389	if (!freed)
	390	atomic_flag_clear(&block->op_list_freed);
	391
	392	/* The block got compiled while the interpreter was running.
	393	* We can free the opcode list now. */
	394	if (block->function && (block->flags & BLOCK_FULLY_TAGGED) &&
	395	!atomic_flag_test_and_set(&block->op_list_freed)) {
	396	pr_debug("Block PC 0x%08x is fully tagged"
	397	" - free opcode list\n", block->pc);
	398
98fa08a5	399	lightrec_free_opcode_list(state, block);
d16005f8 PC	400	block->opcode_list = NULL;
	401	}
	402
	403	return NULL;
	404	}