1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
3 * Copyright (C) 2016-2021 Paul Cercueil <paul@crapouillou.net>
6 #ifndef __LIGHTREC_PRIVATE_H__
7 #define __LIGHTREC_PRIVATE_H__
9 #include "lightning-wrapper.h"
10 #include "lightrec-config.h"
11 #include "disassembler.h"
15 #if ENABLE_THREADED_COMPILER
16 #include <stdatomic.h>
20 #include <immintrin.h>
23 #define ARRAY_SIZE(x) (sizeof(x) ? sizeof(x) / sizeof((x)[0]) : 0)
25 #define GENMASK(h, l) \
26 (((uintptr_t)-1 << (l)) & ((uintptr_t)-1 >> (__WORDSIZE - 1 - (h))))
29 # define likely(x) __builtin_expect(!!(x),1)
30 # define unlikely(x) __builtin_expect(!!(x),0)
32 # define likely(x) (x)
33 # define unlikely(x) (x)
36 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
37 # define LE32TOH(x) __builtin_bswap32(x)
38 # define HTOLE32(x) __builtin_bswap32(x)
39 # define LE16TOH(x) __builtin_bswap16(x)
40 # define HTOLE16(x) __builtin_bswap16(x)
42 # define LE32TOH(x) (x)
43 # define HTOLE32(x) (x)
44 # define LE16TOH(x) (x)
45 # define HTOLE16(x) (x)
49 #define SET_DEFAULT_ELM(table, value) [0 ... ARRAY_SIZE(table) - 1] = value
51 #define SET_DEFAULT_ELM(table, value) [0] = NULL
54 #if __has_attribute(__fallthrough__)
55 # define fallthrough __attribute__((__fallthrough__))
57 # define fallthrough do {} while (0) /* fallthrough */
60 #define container_of(ptr, type, member) \
61 ((type *)((void *)(ptr) - offsetof(type, member)))
64 # define popcount32(x) __popcnt(x)
65 # define clz32(x) _lzcnt_u32(x)
66 # define ctz32(x) _tzcnt_u32(x)
68 # define popcount32(x) __builtin_popcount(x)
69 # define clz32(x) __builtin_clz(x)
70 # define ctz32(x) __builtin_ctz(x)
73 /* Flags for (struct block *)->flags */
74 #define BLOCK_NEVER_COMPILE BIT(0)
75 #define BLOCK_SHOULD_RECOMPILE BIT(1)
76 #define BLOCK_FULLY_TAGGED BIT(2)
77 #define BLOCK_IS_DEAD BIT(3)
78 #define BLOCK_IS_MEMSET BIT(4)
79 #define BLOCK_NO_OPCODE_LIST BIT(5)
80 #define BLOCK_PRELOAD_PC BIT(6)
82 #define RAM_SIZE 0x200000
83 #define BIOS_SIZE 0x80000
85 #define CODE_LUT_SIZE ((RAM_SIZE + BIOS_SIZE) >> 2)
89 #define REG_TEMP (offsetof(struct lightrec_state, temp_reg) / sizeof(u32))
91 /* Definition of jit_state_t (avoids inclusion of <lightning.h>) */
94 typedef struct jit_state jit_state_t;
103 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
112 struct opcode *opcode_list;
113 void (*function)(void);
119 unsigned int code_size;
121 #if ENABLE_THREADED_COMPILER
128 struct lightrec_branch {
129 struct jit_node *branch;
133 struct lightrec_branch_target {
134 struct jit_node *label;
140 C_WRAPPER_RW_GENERIC,
147 struct lightrec_cstate {
148 struct lightrec_state *state;
150 struct lightrec_branch local_branches[512];
151 struct lightrec_branch_target targets[512];
153 unsigned int nb_local_branches;
154 unsigned int nb_targets;
157 struct regcache *reg_cache;
162 struct lightrec_state {
163 struct lightrec_registers regs;
167 uintptr_t wrapper_regs[NUM_TEMPS];
172 u32 old_cycle_counter;
174 struct block *dispatcher, *c_wrapper_block;
175 void *c_wrappers[C_WRAPPERS_COUNT];
176 void *wrappers_eps[C_WRAPPERS_COUNT];
177 struct blockcache *block_cache;
178 struct recompiler *rec;
179 struct lightrec_cstate *cstate;
180 struct reaper *reaper;
182 void (*eob_wrapper_func)(void);
183 void (*interpreter_func)(void);
184 void (*ds_check_func)(void);
185 void (*memset_func)(void);
186 void (*get_next_block)(void);
187 struct lightrec_ops ops;
188 unsigned int nb_precompile;
189 unsigned int nb_compile;
190 unsigned int nb_maps;
191 const struct lightrec_mem_map *maps;
192 uintptr_t offset_ram, offset_bios, offset_scratch, offset_io;
194 _Bool with_32bit_lut;
195 _Bool mirrors_mapped;
199 u32 lightrec_rw(struct lightrec_state *state, union code op, u32 addr,
200 u32 data, u32 *flags, struct block *block, u16 offset);
202 void lightrec_free_block(struct lightrec_state *state, struct block *block);
204 void remove_from_code_lut(struct blockcache *cache, struct block *block);
206 const struct lightrec_mem_map *
207 lightrec_get_map(struct lightrec_state *state, void **host, u32 kaddr);
209 static inline u32 kunseg(u32 addr)
211 if (unlikely(addr >= 0xa0000000))
212 return addr - 0xa0000000;
214 return addr &~ 0x80000000;
217 static inline u32 lut_offset(u32 pc)
220 return ((pc & (BIOS_SIZE - 1)) + RAM_SIZE) >> 2; // BIOS
222 return (pc & (RAM_SIZE - 1)) >> 2; // RAM
225 static inline _Bool is_big_endian(void)
227 return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
230 static inline _Bool lut_is_32bit(const struct lightrec_state *state)
232 return __WORDSIZE == 32 ||
233 (ENABLE_CODE_BUFFER && state->with_32bit_lut);
236 static inline size_t lut_elm_size(const struct lightrec_state *state)
238 return lut_is_32bit(state) ? 4 : sizeof(void *);
241 static inline void ** lut_address(struct lightrec_state *state, u32 offset)
243 if (lut_is_32bit(state))
244 return (void **) ((uintptr_t) state->code_lut + offset * 4);
246 return &state->code_lut[offset];
249 static inline void * lut_read(struct lightrec_state *state, u32 offset)
251 void **lut_entry = lut_address(state, offset);
253 if (lut_is_32bit(state))
254 return (void *)(uintptr_t) *(u32 *) lut_entry;
259 static inline void lut_write(struct lightrec_state *state, u32 offset, void *ptr)
261 void **lut_entry = lut_address(state, offset);
263 if (lut_is_32bit(state))
264 *(u32 *) lut_entry = (u32)(uintptr_t) ptr;
269 static inline u32 get_ds_pc(const struct block *block, u16 offset, s16 imm)
271 u16 flags = block->opcode_list[offset].flags;
273 offset += op_flag_no_ds(flags);
275 return block->pc + ((offset + imm) << 2);
278 static inline u32 get_branch_pc(const struct block *block, u16 offset, s16 imm)
280 u16 flags = block->opcode_list[offset].flags;
282 offset -= op_flag_no_ds(flags);
284 return block->pc + ((offset + imm) << 2);
287 void lightrec_mtc(struct lightrec_state *state, union code op, u8 reg, u32 data);
288 u32 lightrec_mfc(struct lightrec_state *state, union code op);
289 void lightrec_rfe(struct lightrec_state *state);
290 void lightrec_cp(struct lightrec_state *state, union code op);
292 struct lightrec_cstate * lightrec_create_cstate(struct lightrec_state *state);
293 void lightrec_free_cstate(struct lightrec_cstate *cstate);
295 union code lightrec_read_opcode(struct lightrec_state *state, u32 pc);
297 int lightrec_compile_block(struct lightrec_cstate *cstate, struct block *block);
298 void lightrec_free_opcode_list(struct lightrec_state *state,
299 struct opcode *list);
301 unsigned int lightrec_cycles_of_opcode(const struct lightrec_state *state,
304 static inline u8 get_mult_div_lo(union code c)
306 return (OPT_FLAG_MULT_DIV && c.r.rd) ? c.r.rd : REG_LO;
309 static inline u8 get_mult_div_hi(union code c)
311 return (OPT_FLAG_MULT_DIV && c.r.imm) ? c.r.imm : REG_HI;
314 static inline s16 s16_max(s16 a, s16 b)
316 return a > b ? a : b;
319 static inline _Bool block_has_flag(struct block *block, u8 flag)
321 #if ENABLE_THREADED_COMPILER
322 return atomic_load_explicit(&block->flags, memory_order_relaxed) & flag;
324 return block->flags & flag;
328 static inline u8 block_set_flags(struct block *block, u8 mask)
330 #if ENABLE_THREADED_COMPILER
331 return atomic_fetch_or_explicit(&block->flags, mask,
332 memory_order_relaxed);
334 u8 flags = block->flags;
336 block->flags |= mask;
342 static inline u8 block_clear_flags(struct block *block, u8 mask)
344 #if ENABLE_THREADED_COMPILER
345 return atomic_fetch_and_explicit(&block->flags, ~mask,
346 memory_order_relaxed);
348 u8 flags = block->flags;
350 block->flags &= ~mask;
356 static inline _Bool can_sign_extend(s32 value, u8 order)
358 return ((u32)(value >> (order - 1)) + 1) < 2;
361 static inline _Bool can_zero_extend(u32 value, u8 order)
363 return (value >> order) == 0;
366 static inline const struct opcode *
367 get_delay_slot(const struct opcode *list, u16 i)
369 return op_flag_no_ds(list[i].flags) ? &list[i - 1] : &list[i + 1];
372 static inline _Bool lightrec_store_next_pc(void)
374 return NUM_REGS + NUM_TEMPS <= 4;
377 #endif /* __LIGHTREC_PRIVATE_H__ */