1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
7 #include "memmanager.h"
8 #include "lightning-wrapper.h"
14 #define REG_PC (offsetof(struct lightrec_state, curr_pc) / sizeof(u32))
26 struct native_register {
27 bool used, output, extend, extended,
28 zero_extend, zero_extended, locked;
29 s16 emulated_register;
31 enum reg_priority prio;
35 struct lightrec_state *state;
36 struct native_register lightrec_regs[NUM_REGS + NUM_TEMPS];
39 static const char * mips_regs[] = {
43 "a0", "a1", "a2", "a3",
44 "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
45 "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
48 "gp", "sp", "fp", "ra",
52 /* Forward declaration(s) */
53 static void clean_reg(jit_state_t *_jit,
54 struct native_register *nreg, u8 jit_reg, bool clean);
56 const char * lightrec_reg_name(u8 reg)
58 return mips_regs[reg];
61 static inline bool lightrec_reg_is_zero(u8 jit_reg)
63 #if defined(__mips__) || defined(__alpha__) || defined(__riscv)
70 static inline s8 lightrec_get_hardwired_reg(u16 reg)
72 #if defined(__mips__) || defined(__alpha__) || defined(__riscv)
79 static inline u8 lightrec_reg_number(const struct regcache *cache,
80 const struct native_register *nreg)
82 return (u8) (((uintptr_t) nreg - (uintptr_t) cache->lightrec_regs)
86 static inline u8 lightrec_reg_to_lightning(const struct regcache *cache,
87 const struct native_register *nreg)
89 u8 offset = lightrec_reg_number(cache, nreg);
91 if (offset < NUM_REGS)
92 return JIT_V(FIRST_REG + offset);
94 return JIT_R(FIRST_TEMP + offset - NUM_REGS);
97 static inline struct native_register * lightning_reg_to_lightrec(
98 struct regcache *cache, u8 reg)
100 if ((JIT_V0 > JIT_R0 && reg >= JIT_V0) ||
101 (JIT_V0 < JIT_R0 && reg < JIT_R0)) {
103 return &cache->lightrec_regs[reg - JIT_V(FIRST_REG)];
105 return &cache->lightrec_regs[JIT_V(FIRST_REG) - reg];
108 return &cache->lightrec_regs[NUM_REGS + reg - JIT_R(FIRST_TEMP)];
110 return &cache->lightrec_regs[NUM_REGS + JIT_R(FIRST_TEMP) - reg];
114 u8 lightrec_get_reg_in_flags(struct regcache *cache, u8 jit_reg)
116 struct native_register *reg;
119 if (lightrec_reg_is_zero(jit_reg))
120 return REG_EXT | REG_ZEXT;
122 reg = lightning_reg_to_lightrec(cache, jit_reg);
125 if (reg->zero_extended)
131 void lightrec_set_reg_out_flags(struct regcache *cache, u8 jit_reg, u8 flags)
133 struct native_register *reg;
135 if (!lightrec_reg_is_zero(jit_reg)) {
136 reg = lightning_reg_to_lightrec(cache, jit_reg);
137 reg->extend = flags & REG_EXT;
138 reg->zero_extend = flags & REG_ZEXT;
142 static struct native_register * alloc_temp(struct regcache *cache)
144 struct native_register *elm, *nreg = NULL;
145 enum reg_priority best = REG_NB_PRIORITIES;
148 /* We search the register list in reverse order. As temporaries are
149 * meant to be used only in the emitter functions, they can be mapped to
150 * caller-saved registers, as they won't have to be saved back to
152 for (i = ARRAY_SIZE(cache->lightrec_regs); i; i--) {
153 elm = &cache->lightrec_regs[i - 1];
155 if (!elm->used && !elm->locked && elm->prio < best) {
159 if (best == REG_IS_TEMP)
167 static struct native_register * find_mapped_reg(struct regcache *cache,
172 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
173 struct native_register *nreg = &cache->lightrec_regs[i];
174 if ((nreg->prio >= REG_IS_ZERO) &&
175 nreg->emulated_register == reg &&
176 (!out || !nreg->locked))
183 static struct native_register * alloc_in_out(struct regcache *cache,
186 struct native_register *elm, *nreg = NULL;
187 enum reg_priority best = REG_NB_PRIORITIES;
190 /* Try to find if the register is already mapped somewhere */
191 nreg = find_mapped_reg(cache, reg, out);
197 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
198 elm = &cache->lightrec_regs[i];
200 if (!elm->used && !elm->locked && elm->prio < best) {
204 if (best == REG_IS_TEMP)
212 static void lightrec_discard_nreg(struct native_register *nreg)
214 nreg->extended = false;
215 nreg->zero_extended = false;
216 nreg->output = false;
218 nreg->locked = false;
219 nreg->emulated_register = -1;
223 static void lightrec_unload_nreg(struct regcache *cache, jit_state_t *_jit,
224 struct native_register *nreg, u8 jit_reg)
226 clean_reg(_jit, nreg, jit_reg, false);
227 lightrec_discard_nreg(nreg);
230 void lightrec_unload_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
232 if (lightrec_reg_is_zero(jit_reg))
235 lightrec_unload_nreg(cache, _jit,
236 lightning_reg_to_lightrec(cache, jit_reg), jit_reg);
239 u8 lightrec_alloc_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
241 struct native_register *reg;
243 if (lightrec_reg_is_zero(jit_reg))
246 reg = lightning_reg_to_lightrec(cache, jit_reg);
247 lightrec_unload_nreg(cache, _jit, reg, jit_reg);
250 reg->prio = REG_IS_LOADED;
254 u8 lightrec_alloc_reg_temp(struct regcache *cache, jit_state_t *_jit)
257 struct native_register *nreg = alloc_temp(cache);
259 /* No free register, no dirty register to free. */
260 pr_err("No more registers! Abandon ship!\n");
264 jit_reg = lightrec_reg_to_lightning(cache, nreg);
265 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
267 nreg->prio = REG_IS_TEMP;
272 s8 lightrec_get_reg_with_value(struct regcache *cache, intptr_t value)
274 struct native_register *nreg;
277 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++) {
278 nreg = &cache->lightrec_regs[i];
280 if (nreg->prio == REG_IS_TEMP_VALUE && nreg->value == value) {
282 return lightrec_reg_to_lightning(cache, nreg);
289 void lightrec_temp_set_value(struct regcache *cache, u8 jit_reg, intptr_t value)
291 struct native_register *nreg;
293 nreg = lightning_reg_to_lightrec(cache, jit_reg);
295 nreg->prio = REG_IS_TEMP_VALUE;
299 u8 lightrec_alloc_reg_temp_with_value(struct regcache *cache,
300 jit_state_t *_jit, intptr_t value)
304 reg = lightrec_get_reg_with_value(cache, value);
306 reg = lightrec_alloc_reg_temp(cache, _jit);
307 jit_movi((u8)reg, value);
308 lightrec_temp_set_value(cache, (u8)reg, value);
314 u8 lightrec_alloc_reg_out(struct regcache *cache, jit_state_t *_jit,
317 struct native_register *nreg;
321 hw_reg = lightrec_get_hardwired_reg(reg);
325 nreg = alloc_in_out(cache, reg, true);
327 /* No free register, no dirty register to free. */
328 pr_err("No more registers! Abandon ship!\n");
332 jit_reg = lightrec_reg_to_lightning(cache, nreg);
334 /* If we get a dirty register that doesn't correspond to the one
335 * we're requesting, store back the old value */
336 if (nreg->emulated_register != reg)
337 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
341 nreg->emulated_register = reg;
342 nreg->extend = flags & REG_EXT;
343 nreg->zero_extend = flags & REG_ZEXT;
344 nreg->prio = reg ? REG_IS_LOADED : REG_IS_ZERO;
348 u8 lightrec_alloc_reg_in(struct regcache *cache, jit_state_t *_jit,
351 struct native_register *nreg;
356 hw_reg = lightrec_get_hardwired_reg(reg);
360 nreg = alloc_in_out(cache, reg, false);
362 /* No free register, no dirty register to free. */
363 pr_err("No more registers! Abandon ship!\n");
367 jit_reg = lightrec_reg_to_lightning(cache, nreg);
369 /* If we get a dirty register that doesn't correspond to the one
370 * we're requesting, store back the old value */
371 reg_changed = nreg->emulated_register != reg;
373 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
375 if (nreg->prio < REG_IS_LOADED && reg != 0) {
376 s16 offset = lightrec_offset(regs.gpr) + (reg << 2);
378 nreg->zero_extended = flags & REG_ZEXT;
379 nreg->extended = !nreg->zero_extended;
381 /* Load previous value from register cache */
382 if (nreg->zero_extended)
383 jit_ldxi_ui(jit_reg, LIGHTREC_REG_STATE, offset);
385 jit_ldxi_i(jit_reg, LIGHTREC_REG_STATE, offset);
387 nreg->prio = REG_IS_LOADED;
390 /* Clear register r0 before use */
391 if (reg == 0 && nreg->prio != REG_IS_ZERO) {
392 jit_movi(jit_reg, 0);
393 nreg->extended = true;
394 nreg->zero_extended = true;
395 nreg->prio = REG_IS_ZERO;
399 nreg->output = false;
400 nreg->emulated_register = reg;
402 if ((flags & REG_EXT) && !nreg->extended &&
403 (!nreg->zero_extended || !(flags & REG_ZEXT))) {
404 nreg->extended = true;
405 nreg->zero_extended = false;
406 jit_extr_i(jit_reg, jit_reg);
407 } else if (!(flags & REG_EXT) && (flags & REG_ZEXT) &&
408 !nreg->zero_extended) {
409 nreg->zero_extended = true;
410 nreg->extended = false;
411 jit_extr_ui(jit_reg, jit_reg);
417 void lightrec_remap_reg(struct regcache *cache, jit_state_t *_jit,
418 u8 jit_reg, u16 reg_out, bool discard)
420 struct native_register *nreg;
422 lightrec_discard_reg_if_loaded(cache, reg_out);
424 nreg = lightning_reg_to_lightrec(cache, jit_reg);
425 clean_reg(_jit, nreg, jit_reg, !discard);
428 nreg->emulated_register = reg_out;
429 nreg->extend = nreg->extended;
430 nreg->zero_extend = nreg->zero_extended;
433 static bool reg_pc_is_mapped(struct regcache *cache)
435 struct native_register *nreg = lightning_reg_to_lightrec(cache, JIT_V0);
437 return nreg->prio == REG_IS_LOADED && nreg->emulated_register == REG_PC;
440 void lightrec_load_imm(struct regcache *cache,
441 jit_state_t *_jit, u8 jit_reg, u32 pc, u32 imm)
443 s32 delta = imm - pc;
445 if (!reg_pc_is_mapped(cache) || !can_sign_extend(delta, 16))
446 jit_movi(jit_reg, imm);
447 else if (jit_reg != JIT_V0 || delta)
448 jit_addi(jit_reg, JIT_V0, delta);
451 void lightrec_load_next_pc_imm(struct regcache *cache,
452 jit_state_t *_jit, u32 pc, u32 imm)
454 struct native_register *nreg = lightning_reg_to_lightrec(cache, JIT_V0);
457 if (lightrec_store_next_pc())
458 reg = lightrec_alloc_reg_temp(cache, _jit);
460 if (reg_pc_is_mapped(cache)) {
461 /* JIT_V0 contains next PC - so we can overwrite it */
462 lightrec_load_imm(cache, _jit, reg, pc, imm);
464 /* JIT_V0 contains something else - invalidate it */
466 lightrec_unload_reg(cache, _jit, JIT_V0);
471 if (lightrec_store_next_pc()) {
472 jit_stxi_i(lightrec_offset(next_pc), LIGHTREC_REG_STATE, reg);
473 lightrec_free_reg(cache, reg);
475 nreg->prio = REG_IS_LOADED;
476 nreg->emulated_register = -1;
481 void lightrec_load_next_pc(struct regcache *cache, jit_state_t *_jit, u8 reg)
483 struct native_register *nreg_v0, *nreg;
487 if (lightrec_store_next_pc()) {
488 jit_reg = lightrec_alloc_reg_in(cache, _jit, reg, 0);
489 offset = lightrec_offset(next_pc);
490 jit_stxi_i(offset, LIGHTREC_REG_STATE, jit_reg);
491 lightrec_free_reg(cache, jit_reg);
496 /* Invalidate JIT_V0 if it is not mapped to 'reg' */
497 nreg_v0 = lightning_reg_to_lightrec(cache, JIT_V0);
498 if (nreg_v0->prio >= REG_IS_LOADED && nreg_v0->emulated_register != reg)
499 lightrec_unload_nreg(cache, _jit, nreg_v0, JIT_V0);
501 nreg = find_mapped_reg(cache, reg, false);
503 /* Not mapped - load the value from the register cache */
505 offset = lightrec_offset(regs.gpr) + (reg << 2);
506 jit_ldxi_ui(JIT_V0, LIGHTREC_REG_STATE, offset);
508 nreg_v0->prio = REG_IS_LOADED;
509 nreg_v0->emulated_register = reg;
511 } else if (nreg == nreg_v0) {
512 /* The target register 'reg' is mapped to JIT_V0 */
514 if (!nreg->zero_extended)
515 jit_extr_ui(JIT_V0, JIT_V0);
518 /* The target register 'reg' is mapped elsewhere. In that case,
519 * move the register's value to JIT_V0 and re-map it in the
520 * register cache. We can then safely discard the original
521 * mapped register (even if it was dirty). */
523 jit_reg = lightrec_reg_to_lightning(cache, nreg);
524 if (nreg->zero_extended)
525 jit_movr(JIT_V0, jit_reg);
527 jit_extr_ui(JIT_V0, jit_reg);
530 lightrec_discard_nreg(nreg);
533 if (lightrec_store_next_pc()) {
534 jit_stxi_i(lightrec_offset(next_pc),
535 LIGHTREC_REG_STATE, JIT_V0);
537 lightrec_clean_reg(cache, _jit, JIT_V0);
539 nreg_v0->zero_extended = true;
540 nreg_v0->locked = true;
544 static void free_reg(struct native_register *nreg)
546 /* Set output registers as dirty */
547 if (nreg->used && nreg->output && nreg->emulated_register > 0)
548 nreg->prio = REG_IS_DIRTY;
550 nreg->extended = nreg->extend;
551 nreg->zero_extended = nreg->zero_extend;
556 void lightrec_free_reg(struct regcache *cache, u8 jit_reg)
558 if (!lightrec_reg_is_zero(jit_reg))
559 free_reg(lightning_reg_to_lightrec(cache, jit_reg));
562 void lightrec_free_regs(struct regcache *cache)
566 for (i = 0; i < ARRAY_SIZE(cache->lightrec_regs); i++)
567 free_reg(&cache->lightrec_regs[i]);
570 static void clean_reg(jit_state_t *_jit,
571 struct native_register *nreg, u8 jit_reg, bool clean)
573 /* If we get a dirty register, store back the old value */
574 if (nreg->prio == REG_IS_DIRTY) {
575 s16 offset = lightrec_offset(regs.gpr)
576 + (nreg->emulated_register << 2);
578 jit_stxi_i(offset, LIGHTREC_REG_STATE, jit_reg);
581 if (nreg->emulated_register == 0)
582 nreg->prio = REG_IS_ZERO;
584 nreg->prio = REG_IS_LOADED;
589 static void clean_regs(struct regcache *cache, jit_state_t *_jit, bool clean)
593 for (i = 0; i < NUM_REGS; i++) {
594 clean_reg(_jit, &cache->lightrec_regs[i],
595 JIT_V(FIRST_REG + i), clean);
597 for (i = 0; i < NUM_TEMPS; i++) {
598 clean_reg(_jit, &cache->lightrec_regs[i + NUM_REGS],
599 JIT_R(FIRST_TEMP + i), clean);
603 void lightrec_storeback_regs(struct regcache *cache, jit_state_t *_jit)
605 clean_regs(cache, _jit, false);
608 void lightrec_clean_regs(struct regcache *cache, jit_state_t *_jit)
610 clean_regs(cache, _jit, true);
613 bool lightrec_has_dirty_regs(struct regcache *cache)
617 for (i = 0; i < NUM_REGS + NUM_TEMPS; i++)
618 if (cache->lightrec_regs[i].prio == REG_IS_DIRTY)
624 void lightrec_clean_reg(struct regcache *cache, jit_state_t *_jit, u8 jit_reg)
626 struct native_register *reg;
628 if (!lightrec_reg_is_zero(jit_reg)) {
629 reg = lightning_reg_to_lightrec(cache, jit_reg);
630 clean_reg(_jit, reg, jit_reg, true);
634 bool lightrec_reg_is_loaded(struct regcache *cache, u16 reg)
636 return !!find_mapped_reg(cache, reg, false);
639 void lightrec_clean_reg_if_loaded(struct regcache *cache, jit_state_t *_jit,
640 u16 reg, bool unload)
642 struct native_register *nreg;
645 nreg = find_mapped_reg(cache, reg, false);
647 jit_reg = lightrec_reg_to_lightning(cache, nreg);
650 lightrec_unload_nreg(cache, _jit, nreg, jit_reg);
652 clean_reg(_jit, nreg, jit_reg, true);
656 void lightrec_discard_reg_if_loaded(struct regcache *cache, u16 reg)
658 struct native_register *nreg;
660 nreg = find_mapped_reg(cache, reg, false);
662 lightrec_discard_nreg(nreg);
665 struct native_register * lightrec_regcache_enter_branch(struct regcache *cache)
667 struct native_register *backup;
669 backup = lightrec_malloc(cache->state, MEM_FOR_LIGHTREC,
670 sizeof(cache->lightrec_regs));
671 memcpy(backup, &cache->lightrec_regs, sizeof(cache->lightrec_regs));
676 void lightrec_regcache_leave_branch(struct regcache *cache,
677 struct native_register *regs)
679 memcpy(&cache->lightrec_regs, regs, sizeof(cache->lightrec_regs));
680 lightrec_free(cache->state, MEM_FOR_LIGHTREC,
681 sizeof(cache->lightrec_regs), regs);
684 void lightrec_regcache_reset(struct regcache *cache)
686 memset(&cache->lightrec_regs, 0, sizeof(cache->lightrec_regs));
689 void lightrec_preload_pc(struct regcache *cache, jit_state_t *_jit)
691 struct native_register *nreg;
693 /* The block's PC is loaded in JIT_V0 at the start of the block */
694 nreg = lightning_reg_to_lightrec(cache, JIT_V0);
695 nreg->emulated_register = REG_PC;
696 nreg->prio = REG_IS_LOADED;
697 nreg->zero_extended = true;
702 struct regcache * lightrec_regcache_init(struct lightrec_state *state)
704 struct regcache *cache;
706 cache = lightrec_calloc(state, MEM_FOR_LIGHTREC, sizeof(*cache));
710 cache->state = state;
715 void lightrec_free_regcache(struct regcache *cache)
717 return lightrec_free(cache->state, MEM_FOR_LIGHTREC,
718 sizeof(*cache), cache);
721 void lightrec_regcache_mark_live(struct regcache *cache, jit_state_t *_jit)
723 struct native_register *nreg;
727 /* FIXME: GNU Lightning on Windows seems to use our mapped registers as
728 * temporaries. Until the actual bug is found and fixed, unconditionally
729 * mark our registers as live here. */
730 for (i = 0; i < NUM_REGS; i++) {
731 nreg = &cache->lightrec_regs[i];
733 if (nreg->used || nreg->prio > REG_IS_TEMP)
734 jit_live(JIT_V(FIRST_REG + i));
738 for (i = 0; i < NUM_TEMPS; i++) {
739 nreg = &cache->lightrec_regs[NUM_REGS + i];
741 if (nreg->used || nreg->prio > REG_IS_TEMP)
742 jit_live(JIT_R(FIRST_TEMP + i));
745 jit_live(LIGHTREC_REG_STATE);
746 jit_live(LIGHTREC_REG_CYCLE);