[pcsx_rearmed.git] / deps / lightrec / lightrec-private.h

/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2016-2021 Paul Cercueil <paul@crapouillou.net>
 */

#ifndef __LIGHTREC_PRIVATE_H__
#define __LIGHTREC_PRIVATE_H__

#include "lightning-wrapper.h"
#include "lightrec-config.h"
#include "disassembler.h"
#include "lightrec.h"
#include "regcache.h"

#if ENABLE_THREADED_COMPILER
#include <stdatomic.h>
#endif

#ifdef _MSC_BUILD
#include <immintrin.h>
#endif

#define ARRAY_SIZE(x) (sizeof(x) ? sizeof(x) / sizeof((x)[0]) : 0)

#define GENMASK(h, l) \
	(((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))

#ifdef __GNUC__
#	define likely(x)       __builtin_expect(!!(x),1)
#	define unlikely(x)     __builtin_expect(!!(x),0)
#else
#	define likely(x)       (x)
#	define unlikely(x)     (x)
#endif

#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#	define LE32TOH(x)	__builtin_bswap32(x)
#	define HTOLE32(x)	__builtin_bswap32(x)
#	define LE16TOH(x)	__builtin_bswap16(x)
#	define HTOLE16(x)	__builtin_bswap16(x)
#else
#	define LE32TOH(x)	(x)
#	define HTOLE32(x)	(x)
#	define LE16TOH(x)	(x)
#	define HTOLE16(x)	(x)
#endif

#if HAS_DEFAULT_ELM
#define SET_DEFAULT_ELM(table, value) [0 ... ARRAY_SIZE(table) - 1] = value
#else
#define SET_DEFAULT_ELM(table, value) [0] = NULL
#endif

#define fallthrough do {} while (0) /* fall-through */

#define container_of(ptr, type, member) \
	((type *)((void *)(ptr) - offsetof(type, member)))

#ifdef _MSC_BUILD
#	define popcount32(x)	__popcnt(x)
#	define clz32(x)		_lzcnt_u32(x)
#	define ctz32(x)		_tzcnt_u32(x)
#else
#	define popcount32(x)	__builtin_popcount(x)
#	define clz32(x)		__builtin_clz(x)
#	define ctz32(x)		__builtin_ctz(x)
#endif

/* Flags for (struct block *)->flags */
#define BLOCK_NEVER_COMPILE	BIT(0)
#define BLOCK_SHOULD_RECOMPILE	BIT(1)
#define BLOCK_FULLY_TAGGED	BIT(2)
#define BLOCK_IS_DEAD		BIT(3)
#define BLOCK_IS_MEMSET		BIT(4)
#define BLOCK_NO_OPCODE_LIST	BIT(5)

#define RAM_SIZE	0x200000
#define BIOS_SIZE	0x80000

#define CODE_LUT_SIZE	((RAM_SIZE + BIOS_SIZE) >> 2)

#define REG_LO 32
#define REG_HI 33

/* Definition of jit_state_t (avoids inclusion of <lightning.h>) */
struct jit_node;
struct jit_state;
typedef struct jit_state jit_state_t;

struct blockcache;
struct recompiler;
struct regcache;
struct opcode;
struct reaper;

struct u16x2 {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	u16 h, l;
#else
	u16 l, h;
#endif
};

struct block {
	jit_state_t *_jit;
	struct opcode *opcode_list;
	void (*function)(void);
	const u32 *code;
	struct block *next;
	u32 pc;
	u32 hash;
	u32 precompile_date;
	unsigned int code_size;
	u16 nb_ops;
#if ENABLE_THREADED_COMPILER
	_Atomic u8 flags;
#else
	u8 flags;
#endif
};

struct lightrec_branch {
	struct jit_node *branch;
	u32 target;
};

struct lightrec_branch_target {
	struct jit_node *label;
	u32 offset;
};

enum c_wrappers {
	C_WRAPPER_RW,
	C_WRAPPER_RW_GENERIC,
	C_WRAPPER_MFC,
	C_WRAPPER_MTC,
	C_WRAPPER_CP,
	C_WRAPPERS_COUNT,
};

struct lightrec_cstate {
	struct lightrec_state *state;

	struct lightrec_branch local_branches[512];
	struct lightrec_branch_target targets[512];
	unsigned int nb_local_branches;
	unsigned int nb_targets;
	unsigned int cycles;

	struct regcache *reg_cache;
};

struct lightrec_state {
	struct lightrec_registers regs;
	uintptr_t wrapper_regs[NUM_TEMPS];
	u32 next_pc;
	u32 current_cycle;
	u32 target_cycle;
	u32 exit_flags;
	u32 old_cycle_counter;
	struct block *dispatcher, *c_wrapper_block;
	void *c_wrappers[C_WRAPPERS_COUNT];
	void *wrappers_eps[C_WRAPPERS_COUNT];
	struct blockcache *block_cache;
	struct recompiler *rec;
	struct lightrec_cstate *cstate;
	struct reaper *reaper;
	void *tlsf;
	void (*eob_wrapper_func)(void);
	void (*memset_func)(void);
	void (*get_next_block)(void);
	struct lightrec_ops ops;
	unsigned int nb_precompile;
	unsigned int nb_maps;
	const struct lightrec_mem_map *maps;
	uintptr_t offset_ram, offset_bios, offset_scratch, offset_io;
	_Bool with_32bit_lut;
	_Bool mirrors_mapped;
	_Bool invalidate_from_dma_only;
	void *code_lut[];
};

u32 lightrec_rw(struct lightrec_state *state, union code op,
		u32 addr, u32 data, u32 *flags,
		struct block *block);

void lightrec_free_block(struct lightrec_state *state, struct block *block);

void remove_from_code_lut(struct blockcache *cache, struct block *block);

enum psx_map
lightrec_get_map_idx(struct lightrec_state *state, u32 kaddr);

const struct lightrec_mem_map *
lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr);

static inline u32 kunseg(u32 addr)
{
	if (unlikely(addr >= 0xa0000000))
		return addr - 0xa0000000;
	else
		return addr &~ 0x80000000;
}

static inline u32 lut_offset(u32 pc)
{
	if (pc & BIT(28))
		return ((pc & (BIOS_SIZE - 1)) + RAM_SIZE) >> 2; // BIOS
	else
		return (pc & (RAM_SIZE - 1)) >> 2; // RAM
}

static inline _Bool is_big_endian(void)
{
	return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
}

static inline _Bool lut_is_32bit(const struct lightrec_state *state)
{
	return __WORDSIZE == 32 ||
		(ENABLE_CODE_BUFFER && state->with_32bit_lut);
}

static inline size_t lut_elm_size(const struct lightrec_state *state)
{
	return lut_is_32bit(state) ? 4 : sizeof(void *);
}

static inline void ** lut_address(struct lightrec_state *state, u32 offset)
{
	if (lut_is_32bit(state))
		return (void **) ((uintptr_t) state->code_lut + offset * 4);
	else
		return &state->code_lut[offset];
}

static inline void * lut_read(struct lightrec_state *state, u32 offset)
{
	void **lut_entry = lut_address(state, offset);

	if (lut_is_32bit(state))
		return (void *)(uintptr_t) *(u32 *) lut_entry;
	else
		return *lut_entry;
}

static inline void lut_write(struct lightrec_state *state, u32 offset, void *ptr)
{
	void **lut_entry = lut_address(state, offset);

	if (lut_is_32bit(state))
		*(u32 *) lut_entry = (u32)(uintptr_t) ptr;
	else
		*lut_entry = ptr;
}

static inline u32 get_ds_pc(const struct block *block, u16 offset, s16 imm)
{
	u16 flags = block->opcode_list[offset].flags;

	offset += op_flag_no_ds(flags);

	return block->pc + (offset + imm << 2);
}

static inline u32 get_branch_pc(const struct block *block, u16 offset, s16 imm)
{
	u16 flags = block->opcode_list[offset].flags;

	offset -= op_flag_no_ds(flags);

	return block->pc + (offset + imm << 2);
}

void lightrec_mtc(struct lightrec_state *state, union code op, u32 data);
u32 lightrec_mfc(struct lightrec_state *state, union code op);
void lightrec_rfe(struct lightrec_state *state);
void lightrec_cp(struct lightrec_state *state, union code op);

struct lightrec_cstate * lightrec_create_cstate(struct lightrec_state *state);
void lightrec_free_cstate(struct lightrec_cstate *cstate);

union code lightrec_read_opcode(struct lightrec_state *state, u32 pc);

int lightrec_compile_block(struct lightrec_cstate *cstate, struct block *block);
void lightrec_free_opcode_list(struct lightrec_state *state,
			       struct opcode *list);

unsigned int lightrec_cycles_of_opcode(union code code);

static inline u8 get_mult_div_lo(union code c)
{
	return (OPT_FLAG_MULT_DIV && c.r.rd) ? c.r.rd : REG_LO;
}

static inline u8 get_mult_div_hi(union code c)
{
	return (OPT_FLAG_MULT_DIV && c.r.imm) ? c.r.imm : REG_HI;
}

static inline s16 s16_max(s16 a, s16 b)
{
	return a > b ? a : b;
}

static inline _Bool block_has_flag(struct block *block, u8 flag)
{
#if ENABLE_THREADED_COMPILER
	return atomic_load_explicit(&block->flags, memory_order_relaxed) & flag;
#else
	return block->flags & flag;
#endif
}

static inline u8 block_set_flags(struct block *block, u8 mask)
{
#if ENABLE_THREADED_COMPILER
	return atomic_fetch_or_explicit(&block->flags, mask,
					memory_order_relaxed);
#else
	u8 flags = block->flags;

	block->flags |= mask;

	return flags;
#endif
}

static inline u8 block_clear_flags(struct block *block, u8 mask)
{
#if ENABLE_THREADED_COMPILER
	return atomic_fetch_and_explicit(&block->flags, ~mask,
					 memory_order_relaxed);
#else
	u8 flags = block->flags;

	block->flags &= ~mask;

	return flags;
#endif
}

#endif /* __LIGHTREC_PRIVATE_H__ */
Commit	Line	Data
	1	/* SPDX-License-Identifier: LGPL-2.1-or-later */
	2	/*
	3	* Copyright (C) 2016-2021 Paul Cercueil <paul@crapouillou.net>
	4	*/
	5
	6	#ifndef __LIGHTREC_PRIVATE_H__
	7	#define __LIGHTREC_PRIVATE_H__
	8
	9	#include "lightning-wrapper.h"
	10	#include "lightrec-config.h"
	11	#include "disassembler.h"
	12	#include "lightrec.h"
	13	#include "regcache.h"
	14
	15	#if ENABLE_THREADED_COMPILER
	16	#include <stdatomic.h>
	17	#endif
	18
	19	#ifdef _MSC_BUILD
	20	#include <immintrin.h>
	21	#endif
	22
	23	#define ARRAY_SIZE(x) (sizeof(x) ? sizeof(x) / sizeof((x)[0]) : 0)
	24
	25	#define GENMASK(h, l) \
	26	(((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
	27
	28	#ifdef __GNUC__
	29	# define likely(x) __builtin_expect(!!(x),1)
	30	# define unlikely(x) __builtin_expect(!!(x),0)
	31	#else
	32	# define likely(x) (x)
	33	# define unlikely(x) (x)
	34	#endif
	35
	36	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	37	# define LE32TOH(x) __builtin_bswap32(x)
	38	# define HTOLE32(x) __builtin_bswap32(x)
	39	# define LE16TOH(x) __builtin_bswap16(x)
	40	# define HTOLE16(x) __builtin_bswap16(x)
	41	#else
	42	# define LE32TOH(x) (x)
	43	# define HTOLE32(x) (x)
	44	# define LE16TOH(x) (x)
	45	# define HTOLE16(x) (x)
	46	#endif
	47
	48	#if HAS_DEFAULT_ELM
	49	#define SET_DEFAULT_ELM(table, value) [0 ... ARRAY_SIZE(table) - 1] = value
	50	#else
	51	#define SET_DEFAULT_ELM(table, value) [0] = NULL
	52	#endif
	53
	54	#define fallthrough do {} while (0) /* fall-through */
	55
	56	#define container_of(ptr, type, member) \
	57	((type )((void )(ptr) - offsetof(type, member)))
	58
	59	#ifdef _MSC_BUILD
	60	# define popcount32(x) __popcnt(x)
	61	# define clz32(x) _lzcnt_u32(x)
	62	# define ctz32(x) _tzcnt_u32(x)
	63	#else
	64	# define popcount32(x) __builtin_popcount(x)
	65	# define clz32(x) __builtin_clz(x)
	66	# define ctz32(x) __builtin_ctz(x)
	67	#endif
	68
	69	/* Flags for (struct block )->flags /
	70	#define BLOCK_NEVER_COMPILE BIT(0)
	71	#define BLOCK_SHOULD_RECOMPILE BIT(1)
	72	#define BLOCK_FULLY_TAGGED BIT(2)
	73	#define BLOCK_IS_DEAD BIT(3)
	74	#define BLOCK_IS_MEMSET BIT(4)
	75	#define BLOCK_NO_OPCODE_LIST BIT(5)
	76
	77	#define RAM_SIZE 0x200000
	78	#define BIOS_SIZE 0x80000
	79
	80	#define CODE_LUT_SIZE ((RAM_SIZE + BIOS_SIZE) >> 2)
	81
	82	#define REG_LO 32
	83	#define REG_HI 33
	84
	85	/* Definition of jit_state_t (avoids inclusion of <lightning.h>) */
	86	struct jit_node;
	87	struct jit_state;
	88	typedef struct jit_state jit_state_t;
	89
	90	struct blockcache;
	91	struct recompiler;
	92	struct regcache;
	93	struct opcode;
	94	struct reaper;
	95
	96	struct u16x2 {
	97	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	98	u16 h, l;
	99	#else
	100	u16 l, h;
	101	#endif
	102	};
	103
	104	struct block {
	105	jit_state_t *_jit;
	106	struct opcode *opcode_list;
	107	void (*function)(void);
	108	const u32 *code;
	109	struct block *next;
	110	u32 pc;
	111	u32 hash;
	112	u32 precompile_date;
	113	unsigned int code_size;
	114	u16 nb_ops;
	115	#if ENABLE_THREADED_COMPILER
	116	_Atomic u8 flags;
	117	#else
	118	u8 flags;
	119	#endif
	120	};
	121
	122	struct lightrec_branch {
	123	struct jit_node *branch;
	124	u32 target;
	125	};
	126
	127	struct lightrec_branch_target {
	128	struct jit_node *label;
	129	u32 offset;
	130	};
	131
	132	enum c_wrappers {
	133	C_WRAPPER_RW,
	134	C_WRAPPER_RW_GENERIC,
	135	C_WRAPPER_MFC,
	136	C_WRAPPER_MTC,
	137	C_WRAPPER_CP,
	138	C_WRAPPERS_COUNT,
	139	};
	140
	141	struct lightrec_cstate {
	142	struct lightrec_state *state;
	143
	144	struct lightrec_branch local_branches[512];
	145	struct lightrec_branch_target targets[512];
	146	unsigned int nb_local_branches;
	147	unsigned int nb_targets;
	148	unsigned int cycles;
	149
	150	struct regcache *reg_cache;
	151	};
	152
	153	struct lightrec_state {
	154	struct lightrec_registers regs;
	155	uintptr_t wrapper_regs[NUM_TEMPS];
	156	u32 next_pc;
	157	u32 current_cycle;
	158	u32 target_cycle;
	159	u32 exit_flags;
	160	u32 old_cycle_counter;
	161	struct block dispatcher, c_wrapper_block;
	162	void *c_wrappers[C_WRAPPERS_COUNT];
	163	void *wrappers_eps[C_WRAPPERS_COUNT];
	164	struct blockcache *block_cache;
	165	struct recompiler *rec;
	166	struct lightrec_cstate *cstate;
	167	struct reaper *reaper;
	168	void *tlsf;
	169	void (*eob_wrapper_func)(void);
	170	void (*memset_func)(void);
	171	void (*get_next_block)(void);
	172	struct lightrec_ops ops;
	173	unsigned int nb_precompile;
	174	unsigned int nb_maps;
	175	const struct lightrec_mem_map *maps;
	176	uintptr_t offset_ram, offset_bios, offset_scratch, offset_io;
	177	_Bool with_32bit_lut;
	178	_Bool mirrors_mapped;
	179	_Bool invalidate_from_dma_only;
	180	void *code_lut[];
	181	};
	182
	183	u32 lightrec_rw(struct lightrec_state *state, union code op,
	184	u32 addr, u32 data, u32 *flags,
	185	struct block *block);
	186
	187	void lightrec_free_block(struct lightrec_state state, struct block block);
	188
	189	void remove_from_code_lut(struct blockcache cache, struct block block);
	190
	191	enum psx_map
	192	lightrec_get_map_idx(struct lightrec_state *state, u32 kaddr);
	193
	194	const struct lightrec_mem_map *
	195	lightrec_get_map(struct lightrec_state state, void *host, u32 kaddr);
	196
	197	static inline u32 kunseg(u32 addr)
	198	{
	199	if (unlikely(addr >= 0xa0000000))
	200	return addr - 0xa0000000;
	201	else
	202	return addr &~ 0x80000000;
	203	}
	204
	205	static inline u32 lut_offset(u32 pc)
	206	{
	207	if (pc & BIT(28))
	208	return ((pc & (BIOS_SIZE - 1)) + RAM_SIZE) >> 2; // BIOS
	209	else
	210	return (pc & (RAM_SIZE - 1)) >> 2; // RAM
	211	}
	212
	213	static inline _Bool is_big_endian(void)
	214	{
	215	return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
	216	}
	217
	218	static inline _Bool lut_is_32bit(const struct lightrec_state *state)
	219	{
	220	return __WORDSIZE == 32 \|\|
	221	(ENABLE_CODE_BUFFER && state->with_32bit_lut);
	222	}
	223
	224	static inline size_t lut_elm_size(const struct lightrec_state *state)
	225	{
	226	return lut_is_32bit(state) ? 4 : sizeof(void *);
	227	}
	228
	229	static inline void ** lut_address(struct lightrec_state *state, u32 offset)
	230	{
	231	if (lut_is_32bit(state))
	232	return (void *) ((uintptr_t) state->code_lut + offset 4);
	233	else
	234	return &state->code_lut[offset];
	235	}
	236
	237	static inline void * lut_read(struct lightrec_state *state, u32 offset)
	238	{
	239	void **lut_entry = lut_address(state, offset);
	240
	241	if (lut_is_32bit(state))
	242	return (void )(uintptr_t) (u32 *) lut_entry;
	243	else
	244	return *lut_entry;
	245	}
	246
	247	static inline void lut_write(struct lightrec_state state, u32 offset, void ptr)
	248	{
	249	void **lut_entry = lut_address(state, offset);
	250
	251	if (lut_is_32bit(state))
	252	(u32 ) lut_entry = (u32)(uintptr_t) ptr;
	253	else
	254	*lut_entry = ptr;
	255	}
	256
	257	static inline u32 get_ds_pc(const struct block *block, u16 offset, s16 imm)
	258	{
	259	u16 flags = block->opcode_list[offset].flags;
	260
	261	offset += op_flag_no_ds(flags);
	262
	263	return block->pc + (offset + imm << 2);
	264	}
	265
	266	static inline u32 get_branch_pc(const struct block *block, u16 offset, s16 imm)
	267	{
	268	u16 flags = block->opcode_list[offset].flags;
	269
	270	offset -= op_flag_no_ds(flags);
	271
	272	return block->pc + (offset + imm << 2);
	273	}
	274
	275	void lightrec_mtc(struct lightrec_state *state, union code op, u32 data);
	276	u32 lightrec_mfc(struct lightrec_state *state, union code op);
	277	void lightrec_rfe(struct lightrec_state *state);
	278	void lightrec_cp(struct lightrec_state *state, union code op);
	279
	280	struct lightrec_cstate * lightrec_create_cstate(struct lightrec_state *state);
	281	void lightrec_free_cstate(struct lightrec_cstate *cstate);
	282
	283	union code lightrec_read_opcode(struct lightrec_state *state, u32 pc);
	284
	285	int lightrec_compile_block(struct lightrec_cstate cstate, struct block block);
	286	void lightrec_free_opcode_list(struct lightrec_state *state,
	287	struct opcode *list);
	288
	289	unsigned int lightrec_cycles_of_opcode(union code code);
	290
	291	static inline u8 get_mult_div_lo(union code c)
	292	{
	293	return (OPT_FLAG_MULT_DIV && c.r.rd) ? c.r.rd : REG_LO;
	294	}
	295
	296	static inline u8 get_mult_div_hi(union code c)
	297	{
	298	return (OPT_FLAG_MULT_DIV && c.r.imm) ? c.r.imm : REG_HI;
	299	}
	300
	301	static inline s16 s16_max(s16 a, s16 b)
	302	{
	303	return a > b ? a : b;
	304	}
	305
	306	static inline _Bool block_has_flag(struct block *block, u8 flag)
	307	{
	308	#if ENABLE_THREADED_COMPILER
	309	return atomic_load_explicit(&block->flags, memory_order_relaxed) & flag;
	310	#else
	311	return block->flags & flag;
	312	#endif
	313	}
	314
	315	static inline u8 block_set_flags(struct block *block, u8 mask)
	316	{
	317	#if ENABLE_THREADED_COMPILER
	318	return atomic_fetch_or_explicit(&block->flags, mask,
	319	memory_order_relaxed);
	320	#else
	321	u8 flags = block->flags;
	322
	323	block->flags \|= mask;
	324
	325	return flags;
	326	#endif
	327	}
	328
	329	static inline u8 block_clear_flags(struct block *block, u8 mask)
	330	{
	331	#if ENABLE_THREADED_COMPILER
	332	return atomic_fetch_and_explicit(&block->flags, ~mask,
	333	memory_order_relaxed);
	334	#else
	335	u8 flags = block->flags;
	336
	337	block->flags &= ~mask;
	338
	339	return flags;
	340	#endif
	341	}
	342
	343	#endif /* __LIGHTREC_PRIVATE_H__ */