1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
7 #include "memmanager.h"
8 #include "lightning-wrapper.h"
14 #define REG_PC (offsetof(struct lightrec_state, next_pc) / sizeof(u32))
26 struct native_register {
27 bool used, output, extend, extended,
28 zero_extend, zero_extended, locked;
29 s16 emulated_register;
31 enum reg_priority prio;
35 struct lightrec_state *state;
36 struct native_register lightrec_regs[NUM_REGS + NUM_TEMPS];
39 static const char * mips_regs[] = {
43 "a0", "a1", "a2", "a3",
44 "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
45 "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
48 "gp", "sp", "fp", "ra",
52 const char * lightrec_reg_name(u8 reg)
54 return mips_regs[reg];
57 static inline bool lightrec_reg_is_zero(u8 jit_reg)
59 #if defined(__mips__) || defined(__alpha__) || defined(__riscv)
66 static inline s8 lightrec_get_hardwired_reg(u16 reg)
68 #if defined(__mips__) || defined(__alpha__) || defined(__riscv)
75 static inline u8 lightrec_reg_number(const struct regcache *cache,
76 const struct native_register *nreg)
78 return (u8) (((uintptr_t) nreg - (uintptr_t) cache->lightrec_regs)
82 static inline u8 lightrec_reg_to_lightning(const struct regcache *cache,
83 const struct native_register *nreg)
85 u8 offset = lightrec_reg_number(cache, nreg);
87 if (offset < NUM_REGS)
88 return JIT_V(FIRST_REG + offset);
90 return JIT_R(FIRST_TEMP + offset - NUM_REGS);
93 static inline struct native_register * lightning_reg_to_lightrec(
94 struct regcache *cache, u8 reg)
96 if ((JIT_V0 > JIT_R0 && reg >= JIT_V0) ||
97 (JIT_V0 < JIT_R0 && reg < JIT_R0)) {
99 return &cache->lightrec_regs[reg - JIT_V(FIRST_REG)];
101 return &cache->lightrec_regs[JIT_V(FIRST_REG) - reg];
104 return &cache->lightrec_regs[NUM_REGS + reg - JIT_R(FIRST_TEMP)];
106 return &cache->lightrec_regs[NUM_REGS + JIT_R(FIRST_TEMP) - reg];
110 u8 lightrec_get_reg_in_flags(struct regcache *cache, u8 jit_reg)
112 struct native_register *reg;
115 if (lightrec_reg_is_zero(jit_reg))
116 return REG_EXT | REG_ZEXT;
118 reg = lightning_reg_to_lightrec(cache, jit_reg);
121 if (reg->zero_extended)
127 void lightrec_set_reg_out_flags(struct regcache *cache, u8 jit_reg, u8 flags)
129 struct native_register *reg;
131 if (!lightrec_reg_is_zero(jit_reg)) {
132 reg = lightning_reg_to_lightrec(cache, jit_reg);
133 reg->extend = flags & REG_EXT;
134 reg->zero_extend = flags & REG_ZEXT;
138 static struct native_register * alloc_temp(struct regcache *cache)
140 struct native_register *elm, *nreg = NULL;
141 enum reg_priority best = REG_NB_PRIORITIES;
144 /* We search the register list in reverse order. As temporaries are
145 * meant to be used only in the emitter functions, they can be mapped to
146 * caller-saved registers, as they won't have to be saved back to
148 for (i = ARRAY_SIZE(cache->lightrec_regs); i; i--) {
149 elm = &cache->lightrec_regs[i - 1];
151 if (!elm->used && !elm->locked && elm->prio < best) {
155 if (best == REG_IS_TEMP)
163 static struct native_register * find_mapped_reg(struct regcache *cache,
168 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
169 struct native_register *nreg = &cache->lightrec_regs[i];
170 if ((nreg->prio >= REG_IS_ZERO) &&
171 nreg->emulated_register == reg &&
172 (!out || !nreg->locked))
179 static struct native_register * alloc_in_out(struct regcache *cache,
182 struct native_register *elm, *nreg = NULL;
183 enum reg_priority best = REG_NB_PRIORITIES;
186 /* Try to find if the register is already mapped somewhere */
187 nreg = find_mapped_reg(cache, reg, out);
193 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
194 elm = &cache->lightrec_regs[i];
196 if (!elm->used && !elm->locked && elm->prio < best) {
200 if (best == REG_IS_TEMP)
208 static void lightrec_discard_nreg(struct native_register *nreg)
210 nreg->extended = false;
211 nreg->zero_extended = false;
212 nreg->output = false;
214 nreg->locked = false;
215 nreg->emulated_register = -1;
219 static void lightrec_unload_nreg(struct regcache *cache, jit_state_t *_jit,
220 struct native_register *nreg, u8 jit_reg)
222 /* If we get a dirty register, store back the old value */
223 if (nreg->prio == REG_IS_DIRTY) {
224 s16 offset = offsetof(struct lightrec_state, regs.gpr)
225 + (nreg->emulated_register << 2);
227 jit_stxi_i(offset, LIGHTREC_REG_STATE, jit_reg);
230 lightrec_discard_nreg(nreg);
233 void lightrec_unload_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
235 if (lightrec_reg_is_zero(jit_reg))
238 lightrec_unload_nreg(cache, _jit,
239 lightning_reg_to_lightrec(cache, jit_reg), jit_reg);
242 u8 lightrec_alloc_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
244 struct native_register *reg;
246 if (lightrec_reg_is_zero(jit_reg))
249 reg = lightning_reg_to_lightrec(cache, jit_reg);
250 lightrec_unload_nreg(cache, _jit, reg, jit_reg);
253 reg->prio = REG_IS_LOADED;
257 u8 lightrec_alloc_reg_temp(struct regcache *cache, jit_state_t *_jit)
260 struct native_register *nreg = alloc_temp(cache);
262 /* No free register, no dirty register to free. */
263 pr_err("No more registers! Abandon ship!\n");
267 jit_reg = lightrec_reg_to_lightning(cache, nreg);
268 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
270 nreg->prio = REG_IS_TEMP;
275 s8 lightrec_get_reg_with_value(struct regcache *cache, intptr_t value)
277 struct native_register *nreg;
280 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
281 nreg = &cache->lightrec_regs[i];
283 if (nreg->prio == REG_IS_TEMP_VALUE && nreg->value == value) {
285 return lightrec_reg_to_lightning(cache, nreg);
292 void lightrec_temp_set_value(struct regcache *cache, u8 jit_reg, intptr_t value)
294 struct native_register *nreg;
296 nreg = lightning_reg_to_lightrec(cache, jit_reg);
298 nreg->prio = REG_IS_TEMP_VALUE;
302 u8 lightrec_alloc_reg_out(struct regcache *cache, jit_state_t *_jit,
305 struct native_register *nreg;
309 hw_reg = lightrec_get_hardwired_reg(reg);
313 nreg = alloc_in_out(cache, reg, true);
315 /* No free register, no dirty register to free. */
316 pr_err("No more registers! Abandon ship!\n");
320 jit_reg = lightrec_reg_to_lightning(cache, nreg);
322 /* If we get a dirty register that doesn't correspond to the one
323 * we're requesting, store back the old value */
324 if (nreg->emulated_register != reg)
325 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
329 nreg->emulated_register = reg;
330 nreg->extend = flags & REG_EXT;
331 nreg->zero_extend = flags & REG_ZEXT;
332 nreg->prio = reg ? REG_IS_LOADED : REG_IS_ZERO;
336 u8 lightrec_alloc_reg_in(struct regcache *cache, jit_state_t *_jit,
339 struct native_register *nreg;
344 hw_reg = lightrec_get_hardwired_reg(reg);
348 nreg = alloc_in_out(cache, reg, false);
350 /* No free register, no dirty register to free. */
351 pr_err("No more registers! Abandon ship!\n");
355 jit_reg = lightrec_reg_to_lightning(cache, nreg);
357 /* If we get a dirty register that doesn't correspond to the one
358 * we're requesting, store back the old value */
359 reg_changed = nreg->emulated_register != reg;
361 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
363 if (nreg->prio < REG_IS_LOADED && reg != 0) {
364 s16 offset = offsetof(struct lightrec_state, regs.gpr)
367 nreg->zero_extended = flags & REG_ZEXT;
368 nreg->extended = !nreg->zero_extended;
370 /* Load previous value from register cache */
371 if (nreg->zero_extended)
372 jit_ldxi_ui(jit_reg, LIGHTREC_REG_STATE, offset);
374 jit_ldxi_i(jit_reg, LIGHTREC_REG_STATE, offset);
376 nreg->prio = REG_IS_LOADED;
379 /* Clear register r0 before use */
380 if (reg == 0 && nreg->prio != REG_IS_ZERO) {
381 jit_movi(jit_reg, 0);
382 nreg->extended = true;
383 nreg->zero_extended = true;
384 nreg->prio = REG_IS_ZERO;
388 nreg->output = false;
389 nreg->emulated_register = reg;
391 if ((flags & REG_EXT) && !nreg->extended &&
392 (!nreg->zero_extended || !(flags & REG_ZEXT))) {
393 nreg->extended = true;
394 nreg->zero_extended = false;
395 jit_extr_i(jit_reg, jit_reg);
396 } else if (!(flags & REG_EXT) && (flags & REG_ZEXT) &&
397 !nreg->zero_extended) {
398 nreg->zero_extended = true;
399 nreg->extended = false;
400 jit_extr_ui(jit_reg, jit_reg);
406 static bool reg_pc_is_mapped(struct regcache *cache)
408 struct native_register *nreg = lightning_reg_to_lightrec(cache, JIT_V0);
410 return nreg->prio == REG_IS_LOADED && nreg->emulated_register == REG_PC;
413 void lightrec_load_imm(struct regcache *cache,
414 jit_state_t *_jit, u8 jit_reg, u32 pc, u32 imm)
416 s32 delta = imm - pc;
418 if (!reg_pc_is_mapped(cache) || !can_sign_extend(delta, 16))
419 jit_movi(jit_reg, imm);
420 else if (jit_reg != JIT_V0 || delta)
421 jit_addi(jit_reg, JIT_V0, delta);
424 void lightrec_load_next_pc_imm(struct regcache *cache,
425 jit_state_t *_jit, u32 pc, u32 imm)
427 struct native_register *nreg = lightning_reg_to_lightrec(cache, JIT_V0);
429 if (reg_pc_is_mapped(cache)) {
430 /* JIT_V0 contains next PC - so we can overwrite it */
431 lightrec_load_imm(cache, _jit, JIT_V0, pc, imm);
433 /* JIT_V0 contains something else - invalidate it */
434 lightrec_unload_reg(cache, _jit, JIT_V0);
436 jit_movi(JIT_V0, imm);
439 nreg->prio = REG_IS_LOADED;
440 nreg->emulated_register = -1;
444 void lightrec_load_next_pc(struct regcache *cache, jit_state_t *_jit, u8 reg)
446 struct native_register *nreg_v0, *nreg;
450 /* Invalidate JIT_V0 if it is not mapped to 'reg' */
451 nreg_v0 = lightning_reg_to_lightrec(cache, JIT_V0);
452 if (nreg_v0->prio >= REG_IS_LOADED && nreg_v0->emulated_register != reg)
453 lightrec_unload_nreg(cache, _jit, nreg_v0, JIT_V0);
455 nreg = find_mapped_reg(cache, reg, false);
457 /* Not mapped - load the value from the register cache */
459 offset = offsetof(struct lightrec_state, regs.gpr) + (reg << 2);
460 jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE, offset);
462 nreg_v0->prio = REG_IS_LOADED;
463 nreg_v0->emulated_register = reg;
465 } else if (nreg == nreg_v0) {
466 /* The target register 'reg' is mapped to JIT_V0 */
468 if (!nreg->zero_extended)
469 jit_extr_ui(JIT_V0, JIT_V0);
472 /* The target register 'reg' is mapped elsewhere. In that case,
473 * move the register's value to JIT_V0 and re-map it in the
474 * register cache. We can then safely discard the original
475 * mapped register (even if it was dirty). */
477 jit_reg = lightrec_reg_to_lightning(cache, nreg);
478 if (nreg->zero_extended)
479 jit_movr(JIT_V0, jit_reg);
481 jit_extr_ui(JIT_V0, jit_reg);
484 lightrec_discard_nreg(nreg);
487 lightrec_clean_reg(cache, _jit, JIT_V0);
489 nreg_v0->zero_extended = true;
490 nreg_v0->locked = true;
493 static void free_reg(struct native_register *nreg)
495 /* Set output registers as dirty */
496 if (nreg->used && nreg->output && nreg->emulated_register > 0)
497 nreg->prio = REG_IS_DIRTY;
499 nreg->extended = nreg->extend;
500 nreg->zero_extended = nreg->zero_extend;
505 void lightrec_free_reg(struct regcache *cache, u8 jit_reg)
507 if (!lightrec_reg_is_zero(jit_reg))
508 free_reg(lightning_reg_to_lightrec(cache, jit_reg));
511 void lightrec_free_regs(struct regcache *cache)
515 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++)
516 free_reg(&cache->lightrec_regs[i]);
519 static void clean_reg(jit_state_t *_jit,
520 struct native_register *nreg, u8 jit_reg, bool clean)
522 if (nreg->prio == REG_IS_DIRTY) {
523 s16 offset = offsetof(struct lightrec_state, regs.gpr)
524 + (nreg->emulated_register << 2);
526 jit_stxi_i(offset, LIGHTREC_REG_STATE, jit_reg);
529 if (nreg->emulated_register == 0)
530 nreg->prio = REG_IS_ZERO;
532 nreg->prio = REG_IS_LOADED;
537 static void clean_regs(struct regcache *cache, jit_state_t *_jit, bool clean)
541 for (i = 0; i < NUM_REGS; i++) {
542 clean_reg(_jit, &cache->lightrec_regs[i],
543 JIT_V(FIRST_REG + i), clean);
545 for (i = 0; i < NUM_TEMPS; i++) {
546 clean_reg(_jit, &cache->lightrec_regs[i + NUM_REGS],
547 JIT_R(FIRST_TEMP + i), clean);
551 void lightrec_storeback_regs(struct regcache *cache, jit_state_t *_jit)
553 clean_regs(cache, _jit, false);
556 void lightrec_clean_regs(struct regcache *cache, jit_state_t *_jit)
558 clean_regs(cache, _jit, true);
561 bool lightrec_has_dirty_regs(struct regcache *cache)
565 for (i = 0; i < NUM_REGS + NUM_TEMPS; i++)
566 if (cache->lightrec_regs[i].prio == REG_IS_DIRTY)
572 void lightrec_clean_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
574 struct native_register *reg;
576 if (!lightrec_reg_is_zero(jit_reg)) {
577 reg = lightning_reg_to_lightrec(cache, jit_reg);
578 clean_reg(_jit, reg, jit_reg, true);
582 void lightrec_clean_reg_if_loaded(struct regcache *cache, jit_state_t *_jit,
583 u16 reg, bool unload)
585 struct native_register *nreg;
588 nreg = find_mapped_reg(cache, reg, false);
590 jit_reg = lightrec_reg_to_lightning(cache, nreg);
593 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
595 clean_reg(_jit, nreg, jit_reg, true);
599 void lightrec_discard_reg_if_loaded(struct regcache *cache, u16 reg)
601 struct native_register *nreg;
603 nreg = find_mapped_reg(cache, reg, false);
605 lightrec_discard_nreg(nreg);
608 struct native_register * lightrec_regcache_enter_branch(struct regcache *cache)
610 struct native_register *backup;
612 backup = lightrec_malloc(cache->state, MEM_FOR_LIGHTREC,
613 sizeof(cache->lightrec_regs));
614 memcpy(backup, &cache->lightrec_regs, sizeof(cache->lightrec_regs));
619 void lightrec_regcache_leave_branch(struct regcache *cache,
620 struct native_register *regs)
622 memcpy(&cache->lightrec_regs, regs, sizeof(cache->lightrec_regs));
623 lightrec_free(cache->state, MEM_FOR_LIGHTREC,
624 sizeof(cache->lightrec_regs), regs);
627 void lightrec_regcache_reset(struct regcache *cache)
629 memset(&cache->lightrec_regs, 0, sizeof(cache->lightrec_regs));
632 void lightrec_preload_pc(struct regcache *cache)
634 struct native_register *nreg;
636 /* The block's PC is loaded in JIT_V0 at the start of the block */
637 nreg = lightning_reg_to_lightrec(cache, JIT_V0);
638 nreg->emulated_register = REG_PC;
639 nreg->prio = REG_IS_LOADED;
640 nreg->zero_extended = true;
643 struct regcache * lightrec_regcache_init(struct lightrec_state *state)
645 struct regcache *cache;
647 cache = lightrec_calloc(state, MEM_FOR_LIGHTREC, sizeof(*cache));
651 cache->state = state;
656 void lightrec_free_regcache(struct regcache *cache)
658 return lightrec_free(cache->state, MEM_FOR_LIGHTREC,
659 sizeof(*cache), cache);
662 void lightrec_regcache_mark_live(struct regcache *cache, jit_state_t *_jit)
664 struct native_register *nreg;
668 /* FIXME: GNU Lightning on Windows seems to use our mapped registers as
669 * temporaries. Until the actual bug is found and fixed, unconditionally
670 * mark our registers as live here. */
671 for (i = 0; i < NUM_REGS; i++) {
672 nreg = &cache->lightrec_regs[i];
674 if (nreg->used || nreg->prio > REG_IS_TEMP)
675 jit_live(JIT_V(FIRST_REG + i));
679 for (i = 0; i < NUM_TEMPS; i++) {
680 nreg = &cache->lightrec_regs[NUM_REGS + i];
682 if (nreg->used || nreg->prio > REG_IS_TEMP)
683 jit_live(JIT_R(FIRST_TEMP + i));
686 jit_live(LIGHTREC_REG_STATE);
687 jit_live(LIGHTREC_REG_CYCLE);