/*
- * Copyright (C) 2012-2019 Free Software Foundation, Inc.
+ * Copyright (C) 2012-2022 Free Software Foundation, Inc.
*
* This file is part of GNU lightning.
*
static void
_jit_dataset(jit_state_t *_jit);
+#define block_update_set(block, target) _block_update_set(_jit, block, target)
+static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
+
+#define propagate_backward(block) _propagate_backward(_jit, block)
+static void _propagate_backward(jit_state_t*, jit_block_t*);
+
+#define check_block_again() _check_block_again(_jit)
+static jit_bool_t _check_block_again(jit_state_t*);
+
+#define do_setup() _do_setup(_jit)
+static void _do_setup(jit_state_t*);
+
#define jit_setup(block) _jit_setup(_jit, block)
static void
_jit_setup(jit_state_t *_jit, jit_block_t *block);
-#define jit_follow(block, todo) _jit_follow(_jit, block, todo)
+#define do_follow(always) _do_follow(_jit, always)
+static void _do_follow(jit_state_t*, jit_bool_t);
+
+#define jit_follow(block) _jit_follow(_jit, block)
static void
-_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo);
+_jit_follow(jit_state_t *_jit, jit_block_t *block);
#define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
static void
label->link = instr;
}
-void
-_jit_optimize(jit_state_t *_jit)
+static void
+_do_setup(jit_state_t *_jit)
{
- jit_int32_t pass;
- jit_bool_t jump;
- jit_bool_t todo;
- jit_int32_t mask;
- jit_node_t *node;
jit_block_t *block;
jit_word_t offset;
- _jitc->function = NULL;
-
- thread_jumps();
- sequential_labels();
- split_branches();
-
- pass = 0;
-
-second_pass:
/* create initial mapping of live register values
* at the start of a basic block */
for (offset = 0; offset < _jitc->blocks.offset; offset++) {
block = _jitc->blocks.ptr + offset;
- if (!block->label)
+ if (!block->label || block->label->code == jit_code_epilog)
continue;
- if (block->label->code != jit_code_epilog)
- jit_setup(block);
+ jit_setup(block);
}
+}
- /* set live state of registers not referenced in a block, but
- * referenced in a jump target or normal flow */
- do {
- todo = 0;
- for (offset = 0; offset < _jitc->blocks.offset; offset++) {
- block = _jitc->blocks.ptr + offset;
- if (!block->label)
- continue;
- if (block->label->code != jit_code_epilog)
- jit_follow(block, &todo);
- }
- } while (todo);
+static jit_bool_t
+_block_update_set(jit_state_t *_jit,
+ jit_block_t *block, jit_block_t *target)
+{
+ jit_regset_t regmask;
+
+ jit_regset_ior(®mask, &block->reglive, &target->reglive);
+ jit_regset_and(®mask, ®mask, &block->regmask);
+ if (jit_regset_set_p(®mask)) {
+ jit_regset_ior(&block->reglive, &block->reglive, ®mask);
+ jit_regset_and(®mask, &block->reglive, &block->regmask);
+ jit_regset_com(®mask, ®mask);
+ jit_regset_and(&block->regmask, &block->regmask, ®mask);
+ block->again = 1;
+ return (1);
+ }
+ return (0);
+}
- if (pass == 0) {
- todo = 0;
+static void
+_propagate_backward(jit_state_t *_jit, jit_block_t *block)
+{
+ jit_block_t *prev;
+ jit_word_t offset;
- patch_registers();
- if (simplify())
- todo = 1;
+ for (offset = block->label->v.w - 1;
+ offset >= 0; --offset) {
+ prev = _jitc->blocks.ptr + offset;
+ if (!block_update_set(prev, block) ||
+ !(prev->label->flag & jit_flag_head))
+ break;
+ }
+}
- /* figure out labels that are only reached with a jump
- * and is required to do a simple redundant_store removal
- * on jit_beqi below */
- jump = 1;
- for (node = _jitc->head; node; node = node->next) {
- switch (node->code) {
- case jit_code_label:
- if (!jump)
- node->flag |= jit_flag_head;
- break;
- case jit_code_jmpi: case jit_code_jmpr:
- case jit_code_epilog:
- jump = 1;
- break;
- case jit_code_data: case jit_code_note:
- break;
- default:
- jump = 0;
- break;
- }
+static jit_bool_t
+_check_block_again(jit_state_t *_jit)
+{
+ jit_int32_t todo;
+ jit_word_t offset;
+ jit_node_t *node, *label;
+ jit_block_t *block, *target;
+
+ todo = 0;
+ for (offset = 0; offset < _jitc->blocks.offset; offset++) {
+ block = _jitc->blocks.ptr + offset;
+ if (block->again) {
+ todo = 1;
+ break;
}
+ }
+ /* If no block changed state */
+ if (!todo)
+ return (0);
+ do {
+ todo = 0;
+ block = NULL;
for (node = _jitc->head; node; node = node->next) {
- mask = jit_classify(node->code);
- if (mask & jit_cc_a0_reg)
- node->u.w &= ~jit_regno_patch;
- if (mask & jit_cc_a1_reg)
- node->v.w &= ~jit_regno_patch;
- if (mask & jit_cc_a2_reg)
- node->w.w &= ~jit_regno_patch;
- if (node->code == jit_code_beqi) {
- if (redundant_store(node, 1))
+ /* Special jumps that match jit_cc_a0_jmp */
+ if (node->code == jit_code_calli || node->code == jit_code_callr)
+ continue;
+
+ /* Remember current label */
+ if (node->code == jit_code_label ||
+ node->code == jit_code_prolog ||
+ node->code == jit_code_epilog) {
+
+ /* If previous block does not pass through */
+ if (!(node->flag & jit_flag_head))
+ block = NULL;
+
+ target = _jitc->blocks.ptr + node->v.w;
+ if (block && target->again && block_update_set(block, target)) {
+ propagate_backward(block);
todo = 1;
+ }
+ block = target;
}
- else if (node->code == jit_code_bnei) {
- if (redundant_store(node, 0))
+ /* If not the first jmpi */
+ else if (block) {
+ /* If a jump to dynamic address or if a jump to raw address */
+ if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
+ !(node->flag & jit_flag_node))
+ continue;
+ label = node->u.n;
+ /* Mark predecessor needs updating due to target change */
+ target = _jitc->blocks.ptr + label->v.w;
+ if (target->again && block_update_set(block, target)) {
+ propagate_backward(block);
todo = 1;
+ }
}
}
+ }
+ while (todo);
+
+ return (1);
+}
+
+static void
+_do_follow(jit_state_t *_jit, jit_bool_t always)
+{
+ jit_block_t *block;
+ jit_word_t offset;
+
+ /* set live state of registers not referenced in a block, but
+ * referenced in a jump target or normal flow */
+ for (offset = 0; offset < _jitc->blocks.offset; offset++) {
+ block = _jitc->blocks.ptr + offset;
+ if (!block->label || block->label->code == jit_code_epilog)
+ continue;
+ if (always || block->again) {
+ block->again = 0;
+ jit_follow(block);
+ }
+ }
+}
+
+void
+_jit_optimize(jit_state_t *_jit)
+{
+ jit_bool_t jump;
+ jit_bool_t todo;
+ jit_int32_t mask;
+ jit_node_t *node;
+ jit_block_t *block;
+ jit_word_t offset;
- /* If instructions were removed, must recompute state at
- * start of blocks. */
- if (todo) {
- pass = 1;
- goto second_pass;
+ todo = 0;
+ _jitc->function = NULL;
+
+ thread_jumps();
+ sequential_labels();
+ split_branches();
+ do_setup();
+ do_follow(1);
+
+ patch_registers();
+ if (simplify())
+ todo = 1;
+
+ /* Figure out labels that are only reached with a jump
+ * and is required to do a simple redundant_store removal
+ * on jit_beqi below */
+ jump = 1;
+ for (node = _jitc->head; node; node = node->next) {
+ switch (node->code) {
+ case jit_code_label:
+ if (!jump)
+ node->flag |= jit_flag_head;
+ break;
+ case jit_code_jmpi: case jit_code_jmpr:
+ case jit_code_epilog:
+ jump = 1;
+ break;
+ case jit_code_data: case jit_code_note:
+ break;
+ default:
+ jump = 0;
+ break;
}
}
+ for (node = _jitc->head; node; node = node->next) {
+ mask = jit_classify(node->code);
+ if (mask & jit_cc_a0_reg)
+ node->u.w &= ~jit_regno_patch;
+ if (mask & jit_cc_a1_reg)
+ node->v.w &= ~jit_regno_patch;
+ if (mask & jit_cc_a2_reg)
+ node->w.w &= ~jit_regno_patch;
+ if (node->code == jit_code_beqi) {
+ if (redundant_store(node, 1)) {
+ block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
+ block->again = 1;
+ todo = 1;
+ }
+ }
+ else if (node->code == jit_code_bnei) {
+ if (redundant_store(node, 0)) {
+ block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
+ block->again = 1;
+ todo = 1;
+ }
+ }
+ }
+
+ if (!todo)
+ todo = check_block_again();
+
+ /* If instructions were removed or first pass did modify the entry
+ * state of any block */
+ if (todo) {
+ do_setup();
+ todo = 0;
+ do {
+ do_follow(0);
+ /* If any block again has the entry state modified. */
+ todo = check_block_again();
+ } while (todo);
+ }
+
for (node = _jitc->head; node; node = node->next) {
mask = jit_classify(node->code);
if (mask & jit_cc_a0_reg)
#if defined(__sgi)
int mmap_fd;
#endif
+ int mmap_prot, mmap_flags;
if (!_jitc->realize)
jit_realize();
assert(_jit->user_code);
#else
if (!_jit->user_code) {
+ mmap_prot = PROT_READ | PROT_WRITE;
+#if !__OpenBSD__
+ mmap_prot |= PROT_EXEC;
+#endif
+#if __NetBSD__
+ mmap_prot = PROT_MPROTECT(mmap_prot);
+ mmap_flags = 0;
+#else
+ mmap_flags = MAP_PRIVATE;
+#endif
+ mmap_flags |= MAP_ANON;
#if defined(__sgi)
mmap_fd = open("/dev/zero", O_RDWR);
#endif
_jit->code.ptr = mmap(NULL, _jit->code.length,
- PROT_EXEC | PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
+ mmap_prot, mmap_flags, mmap_fd, 0);
assert(_jit->code.ptr != MAP_FAILED);
}
#endif /* !HAVE_MMAP */
_jit->pc.uc = _jit->code.ptr;
for (;;) {
+#if __NetBSD__
+ result = mprotect(_jit->code.ptr, _jit->code.length,
+ PROT_READ | PROT_WRITE);
+ assert(result == 0);
+#endif
if ((code = emit_code()) == NULL) {
_jitc->patches.offset = 0;
for (node = _jitc->head; node; node = node->next) {
jit_free((jit_pointer_t *)&_jitc->data.ptr);
#if HAVE_MMAP
else {
- result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
+ result = mprotect(_jit->data.ptr,
+ _jit->data.length, PROT_READ);
assert(result == 0);
}
if (!_jit->user_code) {
- result = mprotect(_jit->code.ptr, _jit->code.length,
- PROT_READ | PROT_EXEC);
+ length = _jit->pc.uc - _jit->code.ptr;
+# if __riscv && __WORDSIZE == 64
+ /* FIXME should start adding consts at a page boundary */
+ length -= _jitc->consts.hash.count * sizeof(jit_word_t);
+# endif
+ result = mprotect(_jit->code.ptr, length, PROT_READ | PROT_EXEC);
assert(result == 0);
}
#endif /* HAVE_MMAP */
* or normal flow that have a live register not used in this block.
*/
static void
-_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo)
+_jit_follow(jit_state_t *_jit, jit_block_t *block)
{
jit_node_t *node;
jit_block_t *next;
/* Remove from unknown state bitmask. */
jit_regset_com(®temp, ®temp);
jit_regset_and(&block->regmask, &block->regmask, ®temp);
- *todo = 1;
+ block->again = 1;
}
case jit_code_prolog:
case jit_code_epilog:
jit_regset_com(®temp, ®temp);
jit_regset_and(&block->regmask,
&block->regmask, ®temp);
- *todo = 1;
+ block->again = 1;
}
}
else {
jit_node_t *next;
jit_node_t *label;
jit_block_t *block;
+ jit_block_t *blocks;
+ jit_word_t offset;
+ jit_word_t length;
+ length = _jitc->blocks.length;
+ jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
+ if ((node = _jitc->head) &&
+ (node->code == jit_code_label || node->code == jit_code_prolog)) {
+ block = _jitc->blocks.ptr + node->v.w;
+ memcpy(blocks, block, sizeof(jit_block_t));
+ node->v.w = 0;
+ offset = 1;
+ }
+ else
+ offset = 0;
for (node = _jitc->head; node; node = next) {
if ((next = node->next)) {
if (next->code == jit_code_label ||
next->code == jit_code_prolog ||
- next->code == jit_code_epilog)
- continue;
+ next->code == jit_code_epilog) {
+ if (offset >= length) {
+ jit_realloc((jit_pointer_t *)&blocks,
+ length * sizeof(jit_block_t),
+ (length + 16) * sizeof(jit_block_t));
+ length += 16;
+ }
+ block = _jitc->blocks.ptr + next->v.w;
+ memcpy(blocks + offset, block, sizeof(jit_block_t));
+ next->v.w = offset++;
+ }
/* split block on branches */
- if (jit_classify(node->code) & jit_cc_a0_jmp) {
+ else if (jit_classify(node->code) & jit_cc_a0_jmp) {
label = new_node(jit_code_label);
label->next = next;
node->next = label;
- if (_jitc->blocks.offset >= _jitc->blocks.length) {
- jit_word_t length;
-
- length = _jitc->blocks.length + 16;
- jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
- _jitc->blocks.length * sizeof(jit_block_t),
- length * sizeof(jit_block_t));
- _jitc->blocks.length = length;
+ if (offset >= length) {
+ jit_realloc((jit_pointer_t *)&blocks,
+ length * sizeof(jit_block_t),
+ (length + 16) * sizeof(jit_block_t));
+ length += 16;
}
- block = _jitc->blocks.ptr + _jitc->blocks.offset;
+ block = blocks + offset;
block->label = label;
- label->v.w = _jitc->blocks.offset;
+ label->v.w = offset++;
jit_regset_new(&block->reglive);
jit_regset_new(&block->regmask);
- ++_jitc->blocks.offset;
}
}
}
+ jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
+ _jitc->blocks.ptr = blocks;
+ _jitc->blocks.offset = offset;
+ _jitc->blocks.length = length;
}
static jit_bool_t
break;
}
}
+
+ return (result);
}
static jit_bool_t
/* no multiple information, so, if set to a constant,
* prefer to keep that information */
if (value->kind == 0) {
- value->kind = jit_kind_code;
switch (node->code) {
/* no information about signed/unsigned either */
case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
# include "jit_alpha.c"
#elif defined(__riscv)
# include "jit_riscv.c"
+#elif defined(__loongarch__)
+# include "jit_loongarch.c"
#endif
static maybe_unused void